def test_04_upload_handler(self): """ Verify upload_handler API in views.py """ # 1. Login to django server response = self.chrome.post('/explorer/login/', {'username': '******', 'password': '******', 'action': 'login'}) self.assertTrue(response.status_code == 200) # 2. Intialize upload request response = self.chrome.get('/explorer/upload', {'mode' : 'init'}) self.assertTrue(response.status_code == 200) self.assertTrue('ok' in response.content) # Get path to upload file curr = os.getcwd() pathlist = curr.split('/') while pathlist[-1] != 'yang-explorer': pathlist.pop() pathlist.append('default-models') root = '/'.join(pathlist) # 3. Upload file content filelist = ['ietf-inet-types.yang', 'ietf-yang-types.yang', '*****@*****.**'] for _file in filelist: with open(os.path.join(root, _file), 'r') as fp: response = self.chrome.post('/explorer/upload-add', {'Filedata' : fp}) self.assertTrue(response.status_code == 200) self.assertTrue('ok' in response.content) f = response.content.split('<module>')[1].split('</module>')[0] self.upload_files.append(f.strip()) # 4. Compile file for _file in self.upload_files: response = self.chrome.get('/explorer/upload', {'mode':'sync', 'file': _file}) self.assertTrue(response.status_code == 200) self.assertTrue('ok' in response.content) # 5. Commit file response = self.chrome.get('/explorer/upload', {'mode':'commit'}) self.assertTrue(response.status_code == 200) self.assertTrue('ok' in response.content) # logout response = self.chrome.post('/explorer/login/', {'username': '******', 'password': '******', 'action': 'logout'}) self.assertTrue(response.status_code == 200) self.assertTrue('ok' in response.content) # Verify that files are actually uploaded yang_path = ServerSettings.yang_path('demo') cxml_path = ServerSettings.cxml_path('demo') for _file in self.upload_files: # check if yang file is uploaded self.assertTrue(os.path.exists(os.path.join(yang_path, _file))) # check if xml file is created xml_name = _file.split('.yang')[0] + '.xml' self.assertTrue(os.path.exists(os.path.join(cxml_path, xml_name))) print("Test: upload_handler PASSED")
def commit_files(user, session): """ Moves compiled yang moudles to user's yang directory """ directory = ServerSettings.session_path(session) if not os.path.exists(directory): logging.error('Session storage %s does not exist' % directory) return False, None yangdst = ServerSettings.yang_path(user) cxmldst = ServerSettings.cxml_path(user) count = 0 if not os.path.exists(yangdst): logging.debug('Created ' + yangdst) os.makedirs(yangdst) if not os.path.exists(cxmldst): logging.debug('Created ' + cxmldst) os.makedirs(cxmldst) modules = ET.Element('modules') for cxmlpath in glob.glob(os.path.join(directory, '*.xml')): basename = os.path.basename(cxmlpath) if basename == 'dependencies.xml': continue base = os.path.splitext(basename)[0] yang_src_path = os.path.join(directory, base + '.yang') yang_dst_path = os.path.join(yangdst, base + '.yang') cxml_dst_path = os.path.join(cxmldst, base + '.xml') logging.debug('Committing ' + yang_src_path) if os.path.exists(yang_src_path): logging.debug('Commit ' + yang_dst_path) _clean_oldfiles(yangdst, base) _clean_oldfiles(cxmldst, base) os.rename(yang_src_path, yang_dst_path) os.rename(cxmlpath, cxml_dst_path) module = ET.Element('module') module.text = base + '.yang' modules.append(module) count += 1 # There is a update in yang modules, delete existing dependency file # so that it will be recompiled next time if count > 0: session_d = os.path.join(directory, 'dependencies.xml') if os.path.exists(session_d): logging.debug('Moving dependency file ...') os.rename(session_d, os.path.join(yangdst, 'dependencies.xml')) else: logging.debug('Compiling user dependency ...') Compiler.compile_pyimport(user) # added module might affect existing module, recompile them _compile_dependecies(user, [m.text for m in modules], None) logging.debug('Committed ' + str(count) + ' file(s)') return True, modules
def upload_handler(request): """ HTTP Request handler function to upload yang models """ mode = request.GET.get('mode', '') logging.debug(request.method + ':Received upload request .. ' + mode) if not request.user.is_authenticated(): logging.warning('User must be logged in !!') return HttpResponse(Response.error(mode, 'Unauthorized')) if not ServerSettings.user_aware(): if not request.user.has_perm('explorer.delete_yangmodel') or \ not request.user.has_perm('explorer.change_yangmodel'): logging.warning('Unauthorized upload request .. ') return HttpResponse(Response.error(mode, 'User does not have permission to upload !!')) if request.method == 'POST': # create a temporary storage for this session directory = ServerSettings.session_path(request.session.session_key) _file = Uploader.upload_file(request.FILES['Filedata'], directory) if _file is not None: module = ET.Element('module') module.text = _file rval = Response.success('upload', 'ok', xml=module) logging.debug(rval) return HttpResponse(rval) return HttpResponse(Response.error('upload', 'Failed to upload')) elif request.method == 'GET': if mode == 'sync': filename = request.GET.get('file', '') index = request.GET.get('index', '') logging.info('Received sync request for ' + filename + ', index ' + index) success, response = Uploader.sync_file(request.user.username, request.session.session_key, filename, index) if success: return HttpResponse(Response.success(mode, 'ok')) return HttpResponse(Response.error(mode, 'compilation failed', xml=response)) elif mode == 'commit': success, modules = Uploader.commit_files(request.user.username, request.session.session_key) if success: return HttpResponse(Response.success('commit', 'ok', modules)) elif mode == 'init': success, modules = Uploader.get_upload_files(request.user.username, request.session.session_key) if success: return HttpResponse(Response.success(mode, 'ok', modules)) elif mode == 'clear': success, modules = Uploader.clear_upload_files(request.user.username, request.session.session_key) if success: return HttpResponse(Response.success(mode, 'ok', modules)) return HttpResponse(Response.error(mode, 'failed')) return render_to_response('upload.html')
def get_dependencies(username, modules, session): """ return dependencies for given yang models """ session_dir = '' logging.debug("get_dependencies: Target Modules " + str(modules)) if session is not None: session_dir = ServerSettings.session_path(session) dfile = os.path.join(session_dir, 'dependencies.xml') else: dfile = os.path.join(ServerSettings.yang_path(username), 'dependencies.xml') if not os.path.exists(dfile): logging.error('get_dependencies: dependency file %s missing!!', dfile) return [] if session_dir: session_files = [os.path.basename(_file) for _file in glob.glob(os.path.join(session_dir, '*.yang'))] yang_path = ServerSettings.yang_path(username) yang_files = [os.path.basename(_file) for _file in glob.glob(os.path.join(yang_path, '*.yang'))] dmodules = Set([]) dgraph = DYGraph(dfile) for m in modules: module = dgraph.dependency_module(m) if module is None: continue for name in module.imports: dmodules.add(name) for name in module.includes: dmodules.add(name) for name in module.depends: dmodules.add(name) dmodules_list = list(dmodules) deplist = [] for _file in dmodules_list: # prefer freshly uploaded files if session_dir: depfile = _find_matching(_file, session_dir, session_files) else: depfile = _find_matching(_file, yang_path, yang_files) if depfile is not None: deplist.append(depfile) else: logging.warning("get_dependencies: Dependency (%s) not satisfied, compilation will fail !!" % _file) logging.debug("get_dependencies: Computed " + str(deplist)) return deplist
def dependencies_graph(username, modules=[]): depfile = os.path.join(ServerSettings.yang_path(username), 'dependencies.xml') if not os.path.exists(depfile): (rc, msg) = Compiler.compile_pyimport(username, None) if not rc: return rc, msg dgraph = DYGraph(depfile) g = dgraph.digraph([m.text.split('.yang')[0] for m in modules]) if g is None: return ( False, """Failed to generate dependency graph, please make sure that grapviz python package is installed !!""") try: g.render(filename=os.path.join(settings.BASE_DIR, 'static', 'graph')) except: return ( False, """Failed to render dependency graph, please make sure that grapviz binaries (http://www.graphviz.org/Download.php) are installed on the server !!""") return True, g.comment
def download_schema(request, req): """ This API download yang schema from device and bundle it """ logger.debug("Download Schemas") modules = download_yang(request, req) session_dir = ServerSettings.schema_path(request.session.session_key) http_host = request.META["HTTP_HOST"] current = str(datetime.now()) current = current.replace(" ", "-") current = current[: current.rindex(".")] zfname = "schema-" + current + ".zip" zfile = session_dir + "/" + zfname homedir = os.getcwd() os.chdir(session_dir) with ZipFile(zfile, "w") as lz: for f in glob.glob("*.yang"): lz.write(f) os.remove(f) if not lz.namelist(): os.remove(zfile) os.chdir(homedir) url = "\nhttp://" + http_host + "/" + "download/session/" url += request.session.session_key + "/" + zfname return HttpResponse(Response.success("download", msg=url))
def login_handler(request): """ HTTP Request handler function for user login / logout requests """ session = ET.Element('session') if request.POST: action = request.POST['action'] if action == 'login': username = request.POST['username'] password = request.POST['password'] user = authenticate(username=username, password=password) if user is not None and user.is_active: # Correct password, and the user is marked "active" login(request, user) session.text = username else: return HttpResponse(Response.error('login', 'Authentication Failed')) else: try: if request.session.session_key is not None and request.session.session_key != '': session_dir = ServerSettings.session_path(request.session.session_key) if os.path.exists(session_dir): logging.debug('Cleaning ' + session_dir) shutil.rmtree(session_dir) logout(request) except Exception as ex: print(ex.__doc__) print(ex.message) logging.debug('Login success') return HttpResponse(Response.success(action, 'ok', session)) return HttpResponse(Response.error(action, 'Invalid Request'))
def download_schema(request, req): ''' This API download yang schema from device and bundle it ''' logging.debug('Download Schemas') modules = download_yang(request, req) session_dir = ServerSettings.schema_path(request.session.session_key) http_host = request.META['HTTP_HOST'] current = str(datetime.now()) current = current.replace(' ', '-') current = current[:current.rindex('.')] zfname = 'schema-' + current + '.zip' zfile = session_dir + '/' + zfname homedir = os.getcwd() os.chdir(session_dir) with ZipFile(zfile, "w") as lz: for f in glob.glob("*.yang"): lz.write(f) os.remove(f) if not lz.namelist(): os.remove(zfile) os.chdir(homedir) url = '\nhttp://' + http_host + '/' + 'download/session/' + request.session.session_key + '/' + zfname return HttpResponse(Response.success('download', msg=url))
def login_handler(request): """ HTTP Request handler function for user login / logout requests """ if request.POST: action = request.POST['action'] if action == 'login': username = request.POST['username'] password = request.POST['password'] user = authenticate(username=username, password=password) if user is not None and user.is_active: # Correct password, and the user is marked "active" login(request, user) else: return HttpResponse(Response.error('login', 'Authentication Failed')) else: username = '' try: if request.session.session_key is not None and request.session.session_key != '': session_dir = ServerSettings.session_path(request.session.session_key) if os.path.exists(session_dir): logger.debug('Cleaning ' + session_dir) shutil.rmtree(session_dir) logout(request) except: logger.exception("Failed") else: logger.debug('Logout success!!') session = get_session_config(username) return HttpResponse(Response.success(action, 'ok', session)) return HttpResponse(Response.error('unknown', 'Invalid request!!'))
def login_handler(request): """ HTTP Request handler function for user login / logout requests """ session = ET.Element('session') if request.POST: action = request.POST['action'] if action == 'login': username = request.POST['username'] password = request.POST['password'] user = authenticate(username=username, password=password) if user is not None and user.is_active: # Correct password, and the user is marked "active" login(request, user) session.text = username else: return HttpResponse(Response.error('login', 'Authentication Failed')) else: try: if request.session.session_key is not None and request.session.session_key != '': session_dir = ServerSettings.session_path(request.session.session_key) if os.path.exists(session_dir): logging.debug('Cleaning ' + session_dir) shutil.rmtree(session_dir) logout(request) except: logging.exception("Failed") else: logging.debug('Logout success!!') return HttpResponse(Response.success(action, 'ok', session)) return HttpResponse(Response.error('unknown', 'Invalid request!!'))
def get_modules(username): """ Return list of modules available to user + subscribed """ logger.debug("ModuleAdmin.get_modules: enter") modules = ET.Element('modulelist') user = User.objects.filter(username=username) mlist = list() for _file in glob.glob( os.path.join(ServerSettings.yang_path(username), '*.yang')): mlist.append(os.path.basename(_file)) mlist.sort() for fname in mlist: module = ET.Element('module') module.text = os.path.basename(fname) name = module.text.split('.yang')[0] if UserProfile.objects.filter(user=user, module=name).exists(): module.set('subscribed', 'true') modules.append(module) logger.info("ModuleAdmin.get_modules: returning (%d) modules .. exit" % len(modules)) return modules
def compile_pyimport(username, session=None): """ Compile yang model and return tuple (boolean, list-of-errors) """ plugins = os.path.join(settings.BASE_DIR, 'explorer', 'plugins') if not os.path.exists(plugins): logging.error('CXML Plugin directory is missing .. !!') return False, None if subprocess.call(['which', 'pyang']) != 0: logging.error( 'Could not find pyang compiler, please install pyang .. !!') return False, None logging.debug('Rebuilding dependencies for user %s' % username) # build include path includes = [ServerSettings.yang_path(username)] if session is not None: session_dir = ServerSettings.session_path(session) if not os.path.exists(session_dir): logging.error( 'compile_pyimport: Session directory %s not found !!', session_dir) return False, ["Session error !!"] includes.append(session_dir) depfile = os.path.join(session_dir, 'dependencies.xml') else: depfile = os.path.join(includes[0], 'dependencies.xml') target_yangs = [] for yang_dir in includes: for _file in glob.glob(os.path.join(yang_dir, '*.yang')): target_yangs.append(_file) if not target_yangs: logging.debug('compile_pyimport: No yang file found !!') return True, ET.Element('messages') command = [ 'pyang', '-f', 'pyimport', '--ignore-error', '--plugindir', 'explorer/plugins', '-p' ] command += [':'.join(includes)] command += target_yangs return Compiler.invoke_compile(command, depfile)
def cxml_path(username, modulename): _dir = ServerSettings.cxml_path(username) modules = [os.path.basename(_file) for _file in glob.glob(os.path.join(_dir, '*.xml'))] for module in modules: if module == modulename + '.xml': return os.path.join(_dir, module) if module.startswith(modulename + '@'): return os.path.join(_dir, module) return None
def validate_schema(user, name, version): if version is None: fn = os.path.join(ServerSettings.yang_path(user), name + '.yang') else: fn = os.path.join(ServerSettings.yang_path(user), name+'@'+version+'.yang') if os.path.exists(fn): return None dirpath = os.path.join(settings.BASE_DIR, ServerSettings.yang_path(user)) sfile = os.path.basename(fn.split('@')[0]) if not sfile.endswith('.yang'): sfile = sfile + '.yang' for file in os.listdir(dirpath): yfile = os.path.basename(file.split('@')[0]) if not yfile.endswith('.yang'): yfile = yfile + '.yang' if sfile == yfile: return '[out-of-sync]' return '[not-exist]'
def is_browsable(username, module): cxml_path = os.path.join(ServerSettings.cxml_path(username), module + '.xml') browsable = False if os.path.exists(cxml_path): try: root = ET.parse(cxml_path).getroot() if root.find('node'): browsable = True except: logger.error('is_browsable: Exception in parse -> ' + cxml_path) return browsable
def is_browsable(username, module): cxml_path = os.path.join(ServerSettings.cxml_path(username), module + '.xml') browsable = False if os.path.exists(cxml_path): try: root = ET.parse(cxml_path).getroot() if root.find('node'): browsable = True except: logging.error('is_browsable: Excption in parse -> ' + cxml_path) return browsable
def compile_pyimport(username, session=None): ''' Compile yang model and return tuple (boolean, list-of-errors) ''' plugins = os.path.join(settings.BASE_DIR, 'explorer', 'plugins') if not os.path.exists(plugins): logging.error('CXML Plugin directory is missing .. !!') return (False, None) if subprocess.call(['which', 'pyang']) != 0: logging.error('Could not find pyang compiler, please install pyang .. !!') return (False, None) logging.debug('Rebuilding dependencies for user %s' % username) # build include path includes = [ServerSettings.yang_path(username)] if session is not None: session_dir = ServerSettings.session_path(session) if not os.path.exists(session_dir): logging.error('compile_pyimport: Session directory %s not found !!', session_dir) return (False, ["Session error !!"]) includes.append(session_dir) depfile = os.path.join(session_dir, 'dependencies.xml') else: depfile = os.path.join(includes[0], 'dependencies.xml') target_yangs = [] for yang_dir in includes: for _file in glob.glob(os.path.join(yang_dir, '*.yang')): target_yangs.append(_file) if not target_yangs: logging.debug('compile_pyimport: No yang file found !!') return (True, ET.Element('messages')) command = ['pyang', '-f', 'pyimport', '--plugindir', 'explorer/plugins', '-p'] command += [':'.join(includes)] command += target_yangs return Compiler.invoke_compile(command, depfile)
def compile_pyimport(username, session=None): """ Compile yang model and return tuple (boolean, list-of-errors) """ plugins = os.path.join(settings.BASE_DIR, "explorer", "plugins") if not os.path.exists(plugins): logging.error("CXML Plugin directory is missing .. !!") return False, None if subprocess.call(["which", "pyang"]) != 0: logging.error("Could not find pyang compiler, please install pyang .. !!") return False, None logging.debug("Rebuilding dependencies for user %s" % username) # build include path includes = [ServerSettings.yang_path(username)] if session is not None: session_dir = ServerSettings.session_path(session) if not os.path.exists(session_dir): logging.error("compile_pyimport: Session directory %s not found !!", session_dir) return False, ["Session error !!"] includes.append(session_dir) depfile = os.path.join(session_dir, "dependencies.xml") else: depfile = os.path.join(includes[0], "dependencies.xml") target_yangs = [] for yang_dir in includes: for _file in glob.glob(os.path.join(yang_dir, "*.yang")): target_yangs.append(_file) if not target_yangs: logging.debug("compile_pyimport: No yang file found !!") return True, ET.Element("messages") command = ["pyang", "-f", "pyimport", "--plugindir", "explorer/plugins", "-p"] command += [":".join(includes)] command += target_yangs return Compiler.invoke_compile(command, depfile)
def validate_schema(user, name, version): if version is None: fn = os.path.join(ServerSettings.yang_path(user), name + '.yang') else: fn = os.path.join(ServerSettings.yang_path(user), name + '@' + version + '.yang') if os.path.exists(fn): return None dirpath = os.path.join(settings.BASE_DIR, ServerSettings.yang_path(user)) sfile = os.path.basename(fn.split('@')[0]) if not sfile.endswith('.yang'): sfile = sfile + '.yang' for file in os.listdir(dirpath): yfile = os.path.basename(file.split('@')[0]) if not yfile.endswith('.yang'): yfile = yfile + '.yang' if sfile == yfile: return '[out-of-sync]' return '[not-exist]'
def validate_schema(user, name, version): if not version: fn = os.path.join(ServerSettings.yang_path(user), name + ".yang") else: fn = os.path.join(ServerSettings.yang_path(user), name + "@" + version + ".yang") if os.path.exists(fn): return None dirpath = os.path.join(settings.BASE_DIR, ServerSettings.yang_path(user)) sfile = os.path.basename(fn.split("@")[0]) if not sfile.endswith(".yang"): sfile = sfile + ".yang" for file in os.listdir(dirpath): yfile = os.path.basename(file.split("@")[0]) if not yfile.endswith(".yang"): yfile = yfile + ".yang" if sfile == yfile: return "[out-of-sync]" return "[not-exist]"
def clear_upload_files(user, session): """ Delete uploaded yang files which are not committed """ directory = ServerSettings.session_path(session) if not os.path.exists(directory): logging.debug('Session storage %s does not exist' % directory) return False, None modules = ET.Element('modules') for _file in glob.glob(os.path.join(directory, '*')): os.remove(_file) return True, modules
def clear_upload_files(user, session): """ Delete uploaded yang files which are not committed """ directory = ServerSettings.session_path(session) if not os.path.exists(directory): logging.error('Session storage %s does not exist' % directory) return (False, None) modules = ET.Element('modules') for _file in glob.glob(os.path.join(directory, '*')): os.remove(_file) return (True, modules)
def get_upload_files(user, session): """ Get the list of uploaded yang files which are not committed """ modules = ET.Element('modules') directory = ServerSettings.session_path(session) if not os.path.exists(directory): logging.error('Session storage %s does not exist' % directory) return (True, modules) for _file in glob.glob(os.path.join(directory, '*.yang')): module = ET.Element('module') module.text = os.path.basename(_file) modules.append(module) return (True, modules)
def sync_file(user, session, filename, index): """ Compile yang module """ if index == '0': logging.debug('Compiling session dependency ...') (rc, msg) = Compiler.compile_pyimport(user, session) if not rc: return (rc, msg) _file = os.path.join(ServerSettings.session_path(session), filename) if os.path.exists(_file): (rc, msgs) = Compiler.compile_cxml(user, session, _file) else: logging.error('sync_file: File %s not found ' % filename) (rc, msgs) = (False, None) return (rc, msgs)
def get_upload_files(user, session): """ Get the list of uploaded yang files which are not committed """ modules = ET.Element('modules') directory = ServerSettings.session_path(session) if not os.path.exists(directory): logging.error('Session storage %s does not exist' % directory) return True, modules for _file in glob.glob(os.path.join(directory, '*.yang')): module = ET.Element('module') module.text = os.path.basename(_file) modules.append(module) return True, modules
def sync_file(user, session, filename, index): """ Compile yang module """ if index == '0': logging.debug('Compiling session dependency ...') (rc, msg) = Compiler.compile_pyimport(user, session) if not rc: return rc, msg _file = os.path.join(ServerSettings.session_path(session), filename) if os.path.exists(_file): (rc, msgs) = Compiler.compile_cxml(user, session, _file) else: logging.error('sync_file: File %s not found ' % filename) (rc, msgs) = (False, None) return rc, msgs
def get_modulelist(username): """ Return list of modules available to user """ users = User.objects.filter(username=username) if not users: logger.warning("ModuleAdmin.admin_action: Invalid user " + username) return [] modules = [] files = glob.glob(os.path.join(ServerSettings.cxml_path(username), '*.xml')) for _file in files: module = os.path.basename(_file).split('.xml')[0] if UserProfile.objects.filter(user=users[0], module=module).exists(): modules.append(module) return modules
def get_modulelist(username): ''' Return list of modules available to user ''' users = User.objects.filter(username=username) if not users: logging.debug("ModuleAdmin.admin_action: Inavlid user " + username) return [] modules = [] files = glob.glob(os.path.join(ServerSettings.cxml_path(username), '*.xml')) for _file in files: module = os.path.basename(_file).split('.xml')[0] if UserProfile.objects.filter(user=users[0], module=module).exists(): modules.append(module) return modules
def download_yang(request, req): ''' This API download yang schema from device ''' logging.debug('Download Yang Schema') req = req.replace('<metadata>', '') req = req.replace('</metadata>', '') protocol, device, fmt, payload = Adapter.parse_request(req) if device.get('host', None) is None: return HttpResponse(Response.error('download', 'no host info')) session_dir = ServerSettings.schema_path(request.session.session_key) if not os.path.exists(session_dir): os.makedirs(session_dir) if not os.path.exists(session_dir): return HttpResponse(Response.error('download', 'No session directory')) for fname in os.listdir(session_dir): if fname.endswith('.yang'): fn = os.path.join(session_dir, fname) os.remove(fn) modules = ET.Element('modules') reqxml = ET.fromstring(req) schemas = reqxml.find('schemas') for sc in schemas: id = sc.text module = ET.Element('module') get_sc = ET.Element('get-schema') get_sc.set("xmlns", "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring") identifier = ET.Element("identifier") sfile = id.split('@')[0] identifier.text = sfile module.text = id + '.yang' get_sc.append(identifier) rpc.append(get_sc) modules.append(module) schema = Adapter.run_netconf(request.user.username, device, rpc) fname = os.path.join(session_dir, id + '.yang') with open(fname, 'w') as f: f.write(schema[0][0].text) rpc.remove(get_sc) return modules
def get_annotation_tree(): ann_path = ServerSettings.annotation_path(None) filename = os.path.join(ann_path, 'covered.json') if not os.path.exists(filename): return None tree = cache.get('ui_ann', None) if tree is None: tree = XPathTree('/', None) with open(filename, 'r') as f: profile = json.load(f) for line in profile.get('data', []): tree.insert(line, profile.get('annotate', None)) cache.set('ui_ann', tree) else: print 'From cache..' return tree
def download_yang(request, req): ''' This API download yang schema from device ''' logging.debug('Download Yang Schema') req = req.replace('<metadata>', '') req = req.replace('</metadata>', '') protocol, device, fmt, payload = Adapter.parse_request(req) if device.get('host', None) is None: return HttpResponse(Response.error('download', 'no host info')) session_dir = ServerSettings.schema_path(request.session.session_key) if not os.path.exists(session_dir): os.makedirs(session_dir) if not os.path.exists(session_dir): return HttpResponse(Response.error('download', 'No session directory')) for fname in os.listdir(session_dir): if fname.endswith('.yang'): fn = os.path.join(session_dir, fname) os.remove(fn) modules = ET.Element('modules') reqxml = ET.fromstring(req) schemas = reqxml.find('schemas') for sc in schemas: id = sc.text module = ET.Element('module') get_sc = ET.Element('get-schema') get_sc.set("xmlns", "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring") identifier = ET.Element("identifier") sfile = id.split('@')[0] identifier.text = sfile module.text = id+'.yang' get_sc.append(identifier) rpc.append(get_sc) modules.append(module) schema = Adapter.run_netconf(request.user.username, device, rpc) fname = os.path.join(session_dir, id+'.yang') with open(fname, 'w') as f: f.write(schema[0][0].text) rpc.remove(get_sc) return modules
def get_modules(username): ''' Return list of modules available to user + subscribed ''' logging.debug("ModuleAdmin.get_modules: enter") modules = ET.Element('modulelist') user = User.objects.filter(username=username) for _file in glob.glob(os.path.join(ServerSettings.yang_path(username), '*.yang')): module = ET.Element('module') module.text = os.path.basename(_file) name = module.text.split('.yang')[0] if UserProfile.objects.filter(user=user, module=name).exists(): module.set('subscribed', 'true') modules.append(module) logging.debug("ModuleAdmin.get_modules: returning (%d) modules .. exit" % len(modules)) return modules
def create_session_storage(session): """ Create session storage directory for storing session private files Args: session: session id for user Returns: None """ directory = ServerSettings.session_path(session) try: if not os.path.exists(directory): logging.debug('Creating session storage ..') os.makedirs(directory) if not os.path.exists(directory): logging.error('Failed to create session storage ..') return None except: return None return directory
def dependencies_graph(username, modules=[]): depfile = os.path.join(ServerSettings.yang_path(username), 'dependencies.xml') if not os.path.exists(depfile): (rc, msg) = Compiler.compile_pyimport(username, None) if not rc: return (rc, msg) dgraph = DYGraph(depfile) g = dgraph.digraph([m.text.split('.yang')[0] for m in modules]) if g is None: return (False, """Failed to generate dependency graph, please make sure that grapviz python package is installed !!""") try: g.render(filename=os.path.join(settings.BASE_DIR, 'static', 'graph')) except: return (False, """Failed to render dependency graph, please make sure that grapviz binaries (http://www.graphviz.org/Download.php) are installed on the server !!""") return (True, g.comment)
def _compile_dependecies(user, modules, session=None): """ Compile affected modules """ logging.debug('_compile_dependecies: enter') dmodules_list = Compiler.get_dependencies(user, modules, session) if not dmodules_list: logging.debug('_compile_dependecies: no dependency found !!') return # strip file path dmodules = [] for m in dmodules_list: base_m = os.path.basename(m) base_m = os.path.splitext(base_m)[0] if '@' in base_m: base_m = base_m.split('@')[0] dmodules.append(base_m) yangdst = ServerSettings.yang_path(user) for yangfile in glob.glob(os.path.join(yangdst, '*.yang')): basename = os.path.basename(yangfile) # skip dependency module itself if basename in modules: continue base = os.path.splitext(basename)[0] if '@' in base: base = base.split('@')[0] if base in dmodules: # ignore some common files if base in ignore_list: logging.debug('Compile dependency: ignoring ' + base) continue Compiler.compile_cxml(user, None, yangfile) logging.debug('_compile_dependecies: done')
def _compile_dependecies(user, modules, session=None): """ Compile affected modules """ logging.debug('_compile_dependecies: enter') dmodules_list = Compiler.get_dependencies(user, modules, session) if not dmodules_list: logging.debug('_compile_dependecies: no dependency found !!') return #strip file path dmodules = [] for m in dmodules_list: base_m = os.path.basename(m) base_m = os.path.splitext(base_m)[0] if '@' in base_m: base_m = base_m.split('@')[0] dmodules.append(base_m) yangdst = ServerSettings.yang_path(user) for yangfile in glob.glob(os.path.join(yangdst, '*.yang')): basename = os.path.basename(yangfile) #skip dependency module itself if basename in modules: continue base = os.path.splitext(basename)[0] if '@' in base: base = base.split('@')[0] if base in dmodules: # ignore some common files if base in ignore_list: logging.debug('Compile dependency: ignoring ' + base) continue Compiler.compile_cxml(user, None, yangfile) logging.debug('_compile_dependecies: done')
def compile_cxml(username, session, filename): """ Compile yang model and return tuple (boolean, list-of-errors) """ logging.debug("Compiling %s .. !!" % filename) plugins = os.path.join(settings.BASE_DIR, "explorer", "plugins") if not os.path.exists(plugins): logging.error("CXML Plugin directory is missing .. !!") return False, None if subprocess.call(["which", "pyang"]) != 0: logging.error("Could not find pyang compiler, please install pyang .. !!") return False, None basename = os.path.basename(filename) modulename = basename.split(".")[0].strip() session_dir = "" if session is not None: session_dir = ServerSettings.session_path(session) if not os.path.exists(session_dir): logging.error("compile_cxml: Session directory %s not found !!", session_dir) return False, ["Session error !!"] yangfile = os.path.join(session_dir, modulename + ".yang") cxmlfile = os.path.join(session_dir, modulename + ".xml") else: yangfile = os.path.join(ServerSettings.yang_path(username), modulename + ".yang") cxmlfile = os.path.join(ServerSettings.cxml_path(username), modulename + ".xml") # Verify if yang file exists if not os.path.exists(yangfile): logging.debug("compile_cxml: " + yangfile + " not found !!") return False, ["Yang module %s not found on server !!" % modulename] command = ["pyang", "-f", "cxml", "--plugindir", "explorer/plugins", "-p"] # include path for pyang compilation includes = ServerSettings.yang_path(username) if session_dir: includes += ":" + session_dir command.append(includes) # include dependent models command += Compiler.get_dependencies(username, [filename], session) # finally add target module command.append(yangfile) # create a callback to handle empty output def empty_callback(outfile): module = os.path.basename(outfile) module = module.split(".")[0] module = module.split("@")[0] node = ET.Element("node") node.set("name", module) node.set("type", "module") with open(outfile, "w") as fd: fd.write(ET.tostring(node)) logging.debug("compile_cxml: Empty output from pyang, created default cxml!!") return Compiler.invoke_compile(command, cxmlfile, empty_callback)
def test_05_admin_handler(self): """ Verify admin handler functionality """ # 1. Login to django server response = self.chrome.post('/explorer/login/', {'username': '******', 'password': '******', 'action': 'login'}) self.assertTrue(response.status_code == 200) # 2 Subscribe invalid module modules = ET.Element('modules') module = ET.Element('module') module.text = '*****@*****.**' modules.append(module) response = self.chrome.get('/explorer/admin', {'action':'subscribe', 'payload': ET.tostring(modules)}) self.assertTrue(response.status_code == 200) print response.content self.assertTrue('error' in response.content) self.assertTrue('not a main module' in response.content) # 2 Subscribe valid module module.text = '*****@*****.**' response = self.chrome.get('/explorer/admin', {'action':'subscribe', 'payload': ET.tostring(modules)}) self.assertTrue(response.status_code == 200) print response.content self.assertTrue('ok' in response.content) # 3. Verify that returned list correct subscription modulelist = ET.fromstring(response.content).find('modulelist') found = False for m in modulelist: if m.text == '*****@*****.**': found = True self.assertTrue(m.get('subscribed', 'false') == 'true') self.assertTrue(found) # 4. Un-Subscribe ietf-interfaces response = self.chrome.get('/explorer/admin', {'action':'unsubscribe', 'payload': ET.tostring(modules)}) self.assertTrue(response.status_code == 200) print response.content self.assertTrue('ok' in response.content) # 5. Verify that returned list correct subscription modulelist = ET.fromstring(response.content).find('modulelist') found = False for m in modulelist: if m.text == '*****@*****.**': found = True self.assertFalse(m.get('subscribed', 'false') == 'true') self.assertTrue(found) module.text = '*****@*****.**' response = self.chrome.get('/explorer/admin', {'action':'delete', 'payload': ET.tostring(modules)}) self.assertTrue(response.status_code == 200) print response.content self.assertTrue('ok' in response.content) # 6. Verify delete modulelist = ET.fromstring(response.content).find('modulelist') found = False for m in modulelist: self.assertTrue(m.text != '*****@*****.**') _file = module.text yang_path = ServerSettings.yang_path('demo') cxml_path = ServerSettings.cxml_path('demo') # check if yang file is deleted self.assertFalse(os.path.exists(os.path.join(yang_path, _file))) # check if xml file is deleted xml_name = _file.split('.yang')[0] + '.xml' self.assertFalse(os.path.exists(os.path.join(cxml_path, xml_name))) print("Test: admin_handler PASSED")
def compile_cxml(username, session, filename): """ Compile yang model and return tuple (boolean, list-of-errors) """ logging.debug('Compiling %s .. !!' % filename) plugins = os.path.join(settings.BASE_DIR, 'explorer', 'plugins') if not os.path.exists(plugins): logging.error('CXML Plugin directory is missing .. !!') return False, None if subprocess.call(['which', 'pyang']) != 0: logging.error( 'Could not find pyang compiler, please install pyang .. !!') return False, None basename = os.path.basename(filename) modulename = basename.split('.yang')[0].strip() session_dir = '' if session is not None: session_dir = ServerSettings.session_path(session) if not os.path.exists(session_dir): logging.error( 'compile_cxml: Session directory %s not found !!', session_dir) return False, ["Session error !!"] yangfile = os.path.join(session_dir, modulename + '.yang') cxmlfile = os.path.join(session_dir, modulename + '.xml') else: yangfile = os.path.join(ServerSettings.yang_path(username), modulename + '.yang') cxmlfile = os.path.join(ServerSettings.cxml_path(username), modulename + '.xml') # Verify if yang file exists if not os.path.exists(yangfile): logging.debug("compile_cxml: " + yangfile + ' not found !!') return False, [ "Yang module %s not found on server !!" % modulename ] command = [ 'pyang', '-f', 'cxml', '--ignore-error', '--plugindir', 'explorer/plugins', '-p' ] # include path for pyang compilation includes = ServerSettings.yang_path(username) if session_dir: includes += ':' + session_dir command.append(includes) # include dependent models command += Compiler.get_dependencies(username, [filename], session) # finally add target module command.append(yangfile) # create a callback to handle empty output def empty_callback(outfile): module = os.path.basename(outfile) module = module.split('.')[0] module = module.split('@')[0] node = ET.Element('node') node.set('name', module) node.set('type', 'module') with open(outfile, 'w') as fd: fd.write(ET.tostring(node)) logging.debug( 'compile_cxml: Empty output from pyang, created default cxml!!' ) return Compiler.invoke_compile(command, cxmlfile, empty_callback)
def get_dependencies(username, modules, session): """ return dependencies for given yang models """ session_dir = '' logging.debug("get_dependencies: Target Modules " + str(modules)) if session is not None: session_dir = ServerSettings.session_path(session) dfile = os.path.join(session_dir, 'dependencies.xml') else: dfile = os.path.join(ServerSettings.yang_path(username), 'dependencies.xml') if not os.path.exists(dfile): logging.error('get_dependencies: dependency file %s missing!!', dfile) return [] if session_dir: session_files = [ os.path.basename(_file) for _file in glob.glob(os.path.join(session_dir, '*.yang')) ] yang_path = ServerSettings.yang_path(username) yang_files = [ os.path.basename(_file) for _file in glob.glob(os.path.join(yang_path, '*.yang')) ] dmodules = Set([]) dgraph = DYGraph(dfile) for m in modules: module = dgraph.dependency_module(m) if module is None: continue for name in module.imports: dmodules.add(name) for name in module.includes: dmodules.add(name) for name in module.depends: dmodules.add(name) dmodules_list = list(dmodules) deplist = [] for _file in dmodules_list: # prefer freshly uploaded files if session_dir: depfile = _find_matching(_file, session_dir, session_files) else: depfile = _find_matching(_file, yang_path, yang_files) if depfile is not None: deplist.append(depfile) else: logging.warning( "get_dependencies: Dependency (%s) not satisfied, compilation may fail !!" % _file) logging.debug("get_dependencies: Computed " + str(deplist)) return deplist
def admin_action(username, payload, request): logger.info("ModuleAdmin.admin_action: enter (%s -> %s)" % (username, request)) if payload is None: logger.error('ModuleAdmin.admin_action: invalid payload in request !!') return False, "Invalid payload !!" modified = False modules = ET.fromstring(payload) if request == 'graph': return dependencies_graph(username, modules) users = User.objects.filter(username=username) if not users: logger.error("ModuleAdmin.admin_action: invalid user " + username) return False, 'Unknown User %s !!' % username user = users[0] if not ServerSettings.user_aware(): if (request == 'delete') and not user.has_perm('explorer.delete_yangmodel'): return False, 'User %s does not have permission to delete models!!' % username for module in modules: name = module.text.split('.yang')[0] logger.debug("ModuleAdmin.admin_action: %s -> %s" % (request, name)) # delete modules from user profile if request in ['delete', 'unsubscribe']: if UserProfile.objects.filter(user=user, module=name).exists(): profile = UserProfile.objects.filter(user=user, module=name) profile.delete() logger.debug('Module %s deleted for user %s' % (module.text, username)) # delete yang and cxml files for delete request if request == 'delete': for _type in [('cxml', '.xml'), ('yang', '.yang')]: _file = os.path.join('data', 'users', username, _type[0], name + _type[1]) if os.path.exists(_file): os.remove(_file) modified = True logging.debug('Deleted %s (user: %s)' % (_file, username)) if request == 'subscribe': if not is_browsable(username, name): logger.debug('Module %s can not be subscribed ' % (module.text)) continue if not UserProfile.objects.filter(user=user, module=name).exists(): profile = UserProfile(user=user, module=name) profile.save() logger.debug('User %s subscribed to %s module ..' % (username, module.text)) else: logger.debug('User %s already subscribed to %s module ' % (username, module.text)) # if any yang model modified, delete dependency file if modified: _file = os.path.join(ServerSettings.yang_path(username), 'dependencies.xml') if os.path.exists(_file): os.remove(_file) logger.debug('Deleted dependency file %s (user: %s)' % (_file, username)) return True, None
def compile_cxml(username, session, filename): ''' Compile yang model and return tuple (boolean, list-of-errors) ''' logging.debug('Compiling %s .. !!' % filename) plugins = os.path.join(settings.BASE_DIR, 'explorer', 'plugins') if not os.path.exists(plugins): logging.error('CXML Plugin directory is missing .. !!') return (False, None) if subprocess.call(['which', 'pyang']) != 0: logging.error('Could not find pyang compiler, please install pyang .. !!') return (False, None) basename = os.path.basename(filename) modulename = basename.split('.')[0].strip() session_dir = '' if session is not None: session_dir = ServerSettings.session_path(session) if not os.path.exists(session_dir): logging.error('compile_cxml: Session directory %s not found !!', session_dir) return (False, ["Session error !!"]) yangfile = os.path.join(session_dir, modulename + '.yang') cxmlfile = os.path.join(session_dir, modulename + '.xml') else: yangfile = os.path.join(ServerSettings.yang_path(username), modulename + '.yang') cxmlfile = os.path.join(ServerSettings.cxml_path(username), modulename + '.xml') # Verify if yang file exists if not os.path.exists(yangfile): logging.debug("compile_cxml: " + yangfile + ' not found !!') return (False, ["Yang module %s not found on server !!" % modulename]) command = ['pyang', '-f', 'cxml', '--plugindir', 'explorer/plugins', '-p'] # include path for pyang compilation includes = ServerSettings.yang_path(username) if session_dir: includes += ':' + session_dir command.append(includes) # include dependent models command += Compiler.get_dependencies(username, [filename], session) # finally add target module command.append(yangfile) # create a callback to handle empty output def empty_callback(outfile): module = os.path.basename(outfile) module = module.split('.')[0] module = module.split('@')[0] node = ET.Element('node') node.set('name', module) node.set('type', 'module') with open(outfile, 'w') as fd: fd.write(ET.tostring(node)) logging.debug('compile_cxml: Empty output from pyang, created default cxml!!') return Compiler.invoke_compile(command, cxmlfile, empty_callback)
def admin_action(username, payload, request): logging.debug("ModuleAdmin.admin_action: enter (%s -> %s)" % (username, request)) if payload is None: logging.debug('ModuleAdmin.admin_action: Invalid payload in request !!') return (False, "Invalid payload !!") modified = False modules = ET.fromstring(payload) if request == 'graph': return dependencies_graph(username, modules) users = User.objects.filter(username=username) if not users: logging.debug("ModuleAdmin.admin_action: Inavlid user " + username) return (False, 'Unknown User %s !!' % username) user = users[0] if not ServerSettings.user_aware(): if (request == 'delete') and not user.has_perm('explorer.delete_yangmodel'): return (False, 'User %s does not have permission to delete models!!' % username) for module in modules: name = module.text.split('.yang')[0] logging.debug("ModuleAdmin.admin_action: %s -> %s" % (request, name)) # delete modules from user profile if request in ['delete', 'unsubscribe']: if UserProfile.objects.filter(user=user, module=name).exists(): profile = UserProfile.objects.filter(user=user, module=name) profile.delete() logging.debug('Module %s deleted for user %s' % (module.text, username)) # delete yang and cxml files for delete request if request == 'delete': for _type in [('cxml', '.xml'), ('yang', '.yang')]: _file = os.path.join('data', 'users', username, _type[0], name + _type[1]) if os.path.exists(_file): os.remove(_file) modified = True logging.debug('Deleted %s (user: %s)' % (_file, username)) if request == 'subscribe': if not is_browsable(username, name): logging.debug('Module %s can not be subscribed ' % (module.text)) return (False, 'Module %s can not be subscribed, not a main module !!' % name) if not UserProfile.objects.filter(user=user, module=name).exists(): profile = UserProfile(user=user, module=name) profile.save() logging.debug('User %s subscribed to %s module ..' % (username, module.text)) else: logging.debug('User %s already subscribed to %s module ' % (username, module.text)) # if any yang model modified, delete dependency file if modified: _file = os.path.join(ServerSettings.yang_path(username), 'dependencies.xml') if os.path.exists(_file): os.remove(_file) logging.debug('Deleted dependency file %s (user: %s)' % (_file, username)) return (True, None)