def parse_gir(gir_path):
"""Extract everything from a gir file"""
print("Parsing {}".format(gir_path))
parser = XMLParser(encoding="utf-8", recover=True)
content = open(gir_path).read()
root = XML(content, parser)
namespace = root.findall("{%s}namespace" % XMLNS)[0]
namespace_content = extract_namespace(namespace)
return namespace_content
def parse_gir(gir_path):
"""Extract everything from a gir file"""
print("Parsing {}".format(gir_path))
parser = XMLParser(encoding='utf-8', recover=True)
content = open(gir_path).read()
root = XML(content, parser)
namespace = root.findall('{%s}namespace' % XMLNS)[0]
namespace_content = extract_namespace(namespace)
return namespace_content
def process_update(self, xml):
messages = []
xml = XML(xml)
msgs = xml.findall("log/msg")
for msg in msgs:
author = msg.findtext("author")
text = msg.findtext("text")
messages.append(ChatMessage(author, text))
self.messages = messages
def _add_children(self, bus, service):
'''Add the child nodes found by introspection'''
self._children= {};
xml = self._object.Introspect()
data = XML(xml)
# add all child nodes
for child in data.findall('node'):
name = child.get('name')
if name == "/":
continue
child_path = self.object.object_path
# root is reported as /, don't make it //
if child_path != "/":
child_path += "/"
child_path += name
self._children[name] = Dbusitem(bus, service, child_path)
def _add_children(self, bus, service):
'''Add the child nodes found by introspection'''
self._children= {};
xml = self._object.Introspect()
data = XML(xml)
# add all child nodes
for child in data.findall('node'):
name = child.get('name')
if name == "/":
continue
child_path = self.object.object_path
# root is reported as /, don't make it //
if child_path != "/":
child_path += "/"
child_path += name
self._children[name] = Dbusitem(bus, service, child_path)
def _mock_manager_3foobar_warnings(self, *args, **kwargs):
cmd = """
<load-configuration action="set" format="text">
<configuration-set>
delete interfaces ge-0/0/0
delete protocols ospf
delete policy-options prefix-list foo
</configuration-set>
</load-configuration>
"""
rsp_string = """
<rpc-reply>
<load-configuration-results>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
foo boom
</error-message>
</rpc-error>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
boom bar
</error-message>
</rpc-error>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
foo bar
</error-message>
</rpc-error>
<ok/>
</load-configuration-results>
</rpc-reply>
"""
rsp = XML(rsp_string)
errors = []
for err in rsp.findall('.//rpc-error'):
errors.append(RPCError(err))
rpc_error = RPCError(rsp, errs=errors)
raise RpcError(cmd=XML(cmd), rsp=rsp, errs=rpc_error)
def _mock_manager_3foobar_warnings(self, *args, **kwargs):
cmd = """
<load-configuration action="set" format="text">
<configuration-set>
delete interfaces ge-0/0/0
delete protocols ospf
delete policy-options prefix-list foo
</configuration-set>
</load-configuration>
"""
rsp_string = """
<rpc-reply xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" xmlns:junos="http://xml.juniper.net/junos/16.1R4/junos" xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="urn:uuid:1f3dfa00-3434-414a-8aa8-0073590c5812">
<load-configuration-results>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
foo boom
</error-message>
</rpc-error>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
boom bar
</error-message>
</rpc-error>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
foo bar
</error-message>
</rpc-error>
<ok/>
</load-configuration-results>
</rpc-reply>
"""
rsp = XML(rsp_string)
errors = []
for err in rsp.findall('.//' + qualify('rpc-error')):
errors.append(RPCError(err))
raise RPCError(rsp, errs=errors)
def _mock_manager_3foobar_warnings(self, *args, **kwargs):
cmd = """
<load-configuration action="set" format="text">
<configuration-set>
delete interfaces ge-0/0/0
delete protocols ospf
delete policy-options prefix-list foo
</configuration-set>
</load-configuration>
"""
rsp_string = """
<rpc-reply xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" xmlns:junos="http://xml.juniper.net/junos/16.1R4/junos" xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="urn:uuid:1f3dfa00-3434-414a-8aa8-0073590c5812">
<load-configuration-results>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
foo boom
</error-message>
</rpc-error>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
boom bar
</error-message>
</rpc-error>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
foo bar
</error-message>
</rpc-error>
<ok/>
</load-configuration-results>
</rpc-reply>
"""
rsp = XML(rsp_string)
errors = []
for err in rsp.findall('.//'+qualify('rpc-error')):
errors.append(RPCError(err))
raise RPCError(rsp, errs=errors)
def _mock_manager_1snf_warning_1err(self, *args, **kwargs):
cmd = """
<load-configuration action="set" format="text">
<configuration-set>
delete interfaces ge-0/0/0
delete protcols ospf
delete policy-options prefix-list foo
</configuration-set>
</load-configuration>
"""
rsp_string = """
<rpc-reply>
<load-configuration-results>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
statement not found
</error-message>
</rpc-error>
<rpc-error>
<error-type>protocol</error-type>
<error-tag>operation-failed</error-tag>
<error-severity>error</error-severity>
<error-message>syntax error</error-message>
<error-info>
<bad-element>protcols</bad-element>
</error-info>
</rpc-error>
<ok/>
</load-configuration-results>
</rpc-reply>
"""
rsp = XML(rsp_string)
errors = []
for err in rsp.findall('.//rpc-error'):
errors.append(RPCError(err))
rpc_error = RPCError(rsp, errs=errors)
raise RpcError(cmd=XML(cmd), rsp=rsp, errs=rpc_error)
def _mock_manager_1snf_warning_1err(self, *args, **kwargs):
cmd = """
<load-configuration action="set" format="text">
<configuration-set>
delete interfaces ge-0/0/0
delete protcols ospf
delete policy-options prefix-list foo
</configuration-set>
</load-configuration>
"""
rsp_string = """
<rpc-reply xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" xmlns:junos="http://xml.juniper.net/junos/16.1R4/junos" xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="urn:uuid:1f3dfa00-3434-414a-8aa8-0073590c5812">
<load-configuration-results>
<rpc-error>
<error-severity>warning</error-severity>
<error-message>
statement not found
</error-message>
</rpc-error>
<rpc-error>
<error-type>protocol</error-type>
<error-tag>operation-failed</error-tag>
<error-severity>error</error-severity>
<error-message>syntax error</error-message>
<error-info>
<bad-element>protcols</bad-element>
</error-info>
</rpc-error>
<ok/>
</load-configuration-results>
</rpc-reply>
"""
rsp = XML(rsp_string)
errors = []
for err in rsp.findall(".//" + qualify("rpc-error")):
errors.append(RPCError(err))
raise RPCError(rsp, errs=errors)
class CacheResolver(Resolver):
'''Custom resolver to redirect resolution of cached resources back through the cache'''
PICKLESFX = '.history' # type: str
BLANKHIST = {'cache':set([]),'fail':set([])} # type: Dict[str,Set[str]]
#def __init__(self,response,encoding='UTF-8',history=None): # -> None:
def __init__(self,*args, **kwargs): # -> None
self.response = self._testresp(kwargs.get('response') or args[0])
self.encoding = kwargs.get('encoding') or args[1] or 'UTF-8'
self.source = self.response.url
self.history = kwargs.get('history') or args[2] or self._load_hist(self.source)
#self.response = self._testresp(response)
#self.encoding = encoding
#self.history = history or self._load_hist(self.source)
self.doc = XML(validate.SCHMD._extracttxt(self.response,self.encoding))
if self.doc.tag.lower() == 'html':
raise validate.InaccessibleSchemaException('Invalid response, {} is HTML'.format(self.source))
self.target = self.doc.attrib['targetNamespace'] if self.doc.attrib.has_key('targetNamespace') else SL[SLi]
#precache imports
self._precache()
#if not history: # ensures saving only at end of resolve init
self._save_hist()
def _testresp(self,response):
'''Test that the response is a cache type'''
if isinstance(response,CachedResponse):
return response
raise NonCachedResponseException('Provided response object is not from a cached source')
def _precache(self): # -> None:
'''Precache imports and includes'''
for incl in self.doc.findall('xs:include',namespaces=validate.NSX):
for i,ns in enumerate([self.source,self.target]):
if self._getimports(self._cachejoin(ns, incl.attrib['schemaLocation']),'include-{}'.format(i)):
break
for impt in self.doc.findall('xs:import',namespaces=validate.NSX):
for i,ns in enumerate([self.source,self.target,impt.attrib['namespace']]):
if self._getimports(self._cachejoin(ns, impt.attrib['schemaLocation']),'import-{}'.format(i)):
break
def _cachejoin(self,ns,ii4sl): # -> str
'''URL join a base namespace with a import/include path'''
return UR.urljoin(self._slash(ns),ii4sl)
def _getimports(self,url,i): # -> None:
'''import xsd using selectio of urls including targetNamespace, namespace and url of source'''
#for i,url in enumerate(ul):
#hasn't been fetched already or in failed list
if url in self.history['cache']:
pass
#print ('URL {} fetched already'.format(url))
elif url in self.history['fail']:
pass
#print ('Failing URL {} requested'.format(url))
#if url not in [x for v in self.history.values() for x in v]:
else:
try:
#print ('Precaching url {}. {}'.format(i,url))
self._getXMLResponse(url)
#break
return True
except validate.InaccessibleSchemaException as ise:
print (ise)
except XMLSyntaxError as xse:
print ('Cannot parse url {}, {}'.format(i,url))
except Exception as e:
print ('Unexpected Exception {} with {}'.format(e,url))
self.history = CacheResolver._merge(self.history,{'fail':set([url,])})
CachedResponse.RemoveFromCache(self.response.cacheLocation,url)
return False
return True
def _load_hist(self,src=None): # -> Dict:
'''Return fetch/fail history from file or touch/init a new picklefile if reqd'''
hist = self.BLANKHIST
if src and hasattr(self.response,'cacheLocation'):
self.picklefile = '{}/{}{}'.format(self.response.cacheLocation,CachedResponse._hash(src),self.PICKLESFX)
try:
with open(self.picklefile,'rb') as f:
hist = pickle.load(f)
#return self._merge(self.history,pickle.load(open(self.picklefile,'rb')))
except (EOFError, FileNotFoundError) as fnfe:
#touch
with open(self.picklefile, 'ab') as f:
pass
return hist
def _save_hist(self): # -> None:
'''Save the fetched/failed url list into picklefile, merging with existing'''
if hasattr(self,'picklefile'):
hist = self._merge(self.history,self._load_hist())
with open(self.picklefile,'wb') as f:
pickle.dump(hist,f)
@DisplayWrapper.show()
def _getXMLResponse(self,url): # -> None:
resp = validate.SCHMD._request(url)
merge_hist = CacheResolver._merge(self.history,{'cache':set([url,])})
resolver = CacheResolver(resp,self.encoding,merge_hist)
self.history = CacheResolver._merge(self.history,resolver.history)
@staticmethod
def _merge(a,b): # -> Dict:
'''Unidirectional merge from a<-b'''
c = a.copy()
for k in c:
if k in b: c[k].update(b[k])
return c
#return {k:a[k].copy().update(b[k]) if k in b else a[k].copy() for k in a.keys()}
@staticmethod
def _slash(u): # -> str:
'''Adds a trailing / if there isnt one and not a page ref'''
return os.path.join(u,'') if u.rfind('/')>u.rfind('.') else u
def _cached(self,frag): # -> str:
'''Check the system_url fragment matches a url saved in the cache
First match the target+fragment to a history entry
e.g. "http://website/"+"path/schema.xsd" in hist[u1,u2...]
Second search the history for the fragment (could be ambiguous)
e.g. "path/schema.xsd" in hist[u1,u2...]
'''
first = self._cachejoin(self.target, frag)
if first in self.history['cache']:
return first
for u in [h for h in self.history['cache'] if h not in self.history['fail']]:
if frag.lstrip('.') in u: return u
return None
def resolve(self, system_url, public_id, context): # -> str:
'''Override of resolver resolve method that fetches and reads cached pages using the result in a resolve_string call'''
#pprint (self.history)
rstr = None
cached_url = self._cached(system_url)
print (system_url,'->',cached_url)
resp = validate.SCHMD._request(cached_url)
txt = validate.SCHMD._extracttxt(resp,self.encoding)
try:
rstr = self.resolve_string(txt.decode(self.encoding),context) if txt else None
except Exception as e:
print ('resolve_string failed with {}, defaulting'.format(system_url))
return rstr
