def parse_caffe_net(caffe_net):
"""
Create and populate a Topology object, based on a given Caffe protobuf network object
Todo: fix Input assignment
"""
graph = Topology()
# Input BLOBs
for i in range(len(caffe_net.input)):
if len(caffe_net.input_shape) > 0:
graph.add_blob(caffe_net.input[i], caffe_net.input_shape[i].dim,
None)
elif len(caffe_net.input_dim) > 0:
# graph.add_blob(caffe_net.input[i], caffe_net.input_dim[i], None)
graph.add_blob(caffe_net.input[i], caffe_net.input_dim, None)
if len(caffe_net.layer) < 1:
exit(
"Something went wrong - the parser can't find any layers in the network"
)
for layer in caffe_net.layer:
debug('evaluating layer: ' + layer.name)
# filter away layers used only in training phase
phase = 1 # caffe_pb2.Phase.TEST
if phase is not None:
included = False
if len(layer.include) == 0:
included = True
if len(layer.include) > 0 and len(layer.exclude) > 0:
raise ValueError('layer ' + layer.name + ' has both include '
'and exclude specified.')
for layer_phase in layer.include:
included = included or layer_phase.phase == phase
for layer_phase in layer.exclude:
included = included and not layer_phase.phase == phase
if not included:
continue
node_role = 'Producer'
if (len(layer.bottom) == 1 and len(layer.top) == 1
and layer.bottom[0] == layer.top[0]):
# We have an in-place neuron layer.
node_role = 'Modifier'
new_node = graph.add_node(layer.name, layer.type, layer, node_role)
# Iterate over BLOBs consumed by this layer and create edges to them
for caffe_bottom_blob in layer.bottom:
blob = graph.find_blob_by_name(caffe_bottom_blob)
if blob == None:
raise ValueError(layer.name + ' - could not find BLOB:' +
caffe_bottom_blob)
edge = graph.add_edge(src=blob.producer, dst=new_node, blob=blob)
# Add the BLOBs produced by this layer to the topology
for caffe_top_blob in layer.top:
if new_node.type == "Input":
graph.add_blob(caffe_top_blob,
layer.input_param.shape[0].dim,
producer=new_node)
else:
graph.add_blob(caffe_top_blob, None, producer=new_node)
# Add fake output edges
output_blobs = graph.find_output_blobs()
for blob_name in output_blobs:
blob = graph.find_blob_by_name(blob_name)
graph.add_edge(src=blob.producer, dst=None, blob=blob)
return graph
class SC16:
SCHEMAS = {
'networkresources':
'http://unis.crest.iu.edu/schema/20151104/networkresource#',
'nodes': 'http://unis.crest.iu.edu/schema/20151104/node#',
'domains': 'http://unis.crest.iu.edu/schema/20151104/domain#',
'ports': 'http://unis.crest.iu.edu/schema/20151104/port#',
'links': 'http://unis.crest.iu.edu/schema/20151104/link#',
'paths': 'http://unis.crest.iu.edu/schema/20151104/path#',
'networks': 'http://unis.crest.iu.edu/schema/20151104/network#',
'topologies': 'http://unis.crest.iu.edu/schema/20151104/topology#',
'services': 'http://unis.crest.iu.edu/schema/20151104/service#',
'blipp': 'http://unis.crest.iu.edu/schema/20151104/blipp#',
'metadata': 'http://unis.crest.iu.edu/schema/20151104/metadata#',
'datum': 'http://unis.crest.iu.edu/schema/20151104/datum#',
'data': 'http://unis.crest.iu.edu/schema/20151104/data#',
'ipports': 'http://unis.crest.iu.edu/schema/ext/ipport/1/ipport#'
}
domain_list = [{
'domain_name':
'IU',
'domain_ref':
'',
'nodes': [{
'node_name': 'IU',
'node_ref': '',
'links': [{
'source': 'IU',
'sink': 'CHIC'
}]
}]
}]
def __init__(self, host_name_unis, port_number_unis):
self.topology_object = Topology()
self.unis_uri = "http://" + host_name_unis + ":" + port_number_unis + "/"
self.clean_all()
def clean_all(self):
print("Starting the initial CLEANUP PROCESS")
self.clean_domains()
self.clean_topology()
self.clean_nodes()
self.delete_ports()
self.delete_links()
# self.create_nodes()
# self.update_node_refs()
# self.create_domains()
# self.update_domain_refs()
def clean_topology(self):
print("Deleting the Topology")
topology_uri = self.unis_uri + "topologies"
topology_list = coreapi.get(topology_uri)
for topo_dict in topology_list:
print topo_dict['selfRef']
# requests.delete(node_dict['selfRef'])
del_uri = topology_uri + "/" + topo_dict['selfRef'].split("/")[4]
requests.delete(del_uri)
def clean_domains(self):
print("Deleting the Domains")
domains_uri = self.unis_uri + "domains"
domain_list = coreapi.get(domains_uri)
for domain_dict in domain_list:
print domain_dict['selfRef']
# requests.delete(node_dict['selfRef'])
del_uri = domains_uri + "/" + domain_dict['selfRef'].split("/")[4]
requests.delete(del_uri)
def clean_nodes(self):
print("Deleting the nodes")
nodes_uri = self.unis_uri + "nodes"
nodes_list = coreapi.get(nodes_uri)
for node_dict in nodes_list:
print node_dict['selfRef']
# requests.delete(node_dict['selfRef'])
del_uri = nodes_uri + "/" + node_dict['selfRef'].split("/")[4]
requests.delete(del_uri)
def delete_ports(self):
print("Deleting the ports")
ports_uri = self.unis_uri + "ports"
ports_list = coreapi.get(ports_uri)
for port_dict in ports_list:
print port_dict['selfRef']
# requests.delete(port_dict['selfRef'])
del_uri = ports_uri + "/" + port_dict['selfRef'].split("/")[4]
print("Delete:" + del_uri)
requests.delete(del_uri)
def delete_links(self):
print("Deleting the links")
links_uri = self.unis_uri + "links"
links_list = coreapi.get(links_uri)
for link_dict in links_list:
print link_dict['selfRef']
# requests.delete(port_dict['selfRef'])
del_uri = links_uri + "/" + link_dict['selfRef'].split("/")[4]
print("Delete:" + del_uri)
requests.delete(del_uri)
def create_domains(self):
domain_dict_list = []
domains_uri = self.unis_uri + "domains"
for domain in self.domain_list:
domain_dict = dict()
domain_dict["$schema"] = self.SCHEMAS['domains']
domain_dict["name"] = domain['domain_name']
nodes_dict_list = []
nodes_list = self.get_nodes_from_domain_list(domain['domain_name'])
for node in nodes_list:
nodes_dict_list.append({'name': 'IU'})
domain_dict["nodes"] = nodes_dict_list
domain_dict_list.append(domain_dict)
json_data = json.dumps(domain_dict_list)
print("Domain URI::" + domains_uri)
print("***** DOMAINS JSON DATA:" + json_data)
requests.post(domains_uri, data=json_data)
def get_nodes_from_domain_list(self, domain_name):
for domain in self.domain_list:
if domain['domain_name'] == domain_name:
return domain['nodes']
def update_domain_refs(self):
domains_uri = self.unis_uri + "domains"
domains_list = coreapi.get(domains_uri)
for domain in domains_list:
print domain['name']
print domain['selfRef']
# node_object = Node(domain['name'])
self.add_domain_ref_to_list(domain['name'], domain['selfRef'])
# self.topology_object.add_node(self.getId(check_node['selfRef']), node_object)
print "\n"
print(self.domain_list)
# self.topology_object.display_topology()
def add_domain_ref_to_list(self, domain_name, domain_ref):
for domain in self.domain_list:
if domain['domain_name'] == domain_name:
domain['domain_ref'] = domain_ref
return
def create_nodes(self):
nodes_uri = self.unis_uri + "nodes"
nodes = []
for domain in self.domain_list:
for node_dict in domain['nodes']:
node = dict()
node["$schema"] = self.SCHEMAS['nodes']
node["name"] = node_dict['node_name']
# print node_dict['node-id']
nodes.append(node)
print "::FINAL JSON::"
json_data = json.dumps(nodes)
print("NODE URI::" + nodes_uri)
print("JSON DATA:" + json_data)
requests.post(nodes_uri, data=json_data)
def update_node_refs(self):
nodes_uri = self.unis_uri + "nodes"
nodes_list = coreapi.get(nodes_uri)
for check_node in nodes_list:
print check_node['name']
print check_node['selfRef']
node_object = Node(check_node['name'])
self.topology_object.add_node(self.getId(check_node['selfRef']),
node_object)
self.add_node_ref_to_domain_list(check_node['selfRef'],
check_node['name'])
print "\n"
self.topology_object.display_topology()
def add_node_ref_to_domain_list(self, node_ref, node_name):
for domain in self.domain_list:
for node in domain['nodes']:
if node['node_name'] == node_name:
node['node_ref'] = node_ref
return
def getId(self, ref_url):
"""
http://10.10.0.135:8888/nodes/56f88569e1382308b0b6a2ea will return 56f88569e1382308b0b6a2ea
:param ref_url:
:return:
"""
print("getId::" + ref_url)
return ref_url.split("/")[4]
def create_ref_url(self, type, id=None):
"""
It will build the url from type and id
:param type: ports or nodes
:param id:
:return:
"""
if id is None:
return self.unis_uri + type
return self.unis_uri + type + "/" + id
class RdfDecoder():
SCHEMAS = {
'networkresources': 'http://unis.crest.iu.edu/schema/20151104/networkresource#',
'nodes': 'http://unis.crest.iu.edu/schema/20151104/node#',
'domains': 'http://unis.crest.iu.edu/schema/20151104/domain#',
'ports': 'http://unis.crest.iu.edu/schema/20151104/port#',
'links': 'http://unis.crest.iu.edu/schema/20151104/link#',
'paths': 'http://unis.crest.iu.edu/schema/20151104/path#',
'networks': 'http://unis.crest.iu.edu/schema/20151104/network#',
'topologies': 'http://unis.crest.iu.edu/schema/20151104/topology#',
'services': 'http://unis.crest.iu.edu/schema/20151104/service#',
'blipp': 'http://unis.crest.iu.edu/schema/20151104/blipp#',
'metadata': 'http://unis.crest.iu.edu/schema/20151104/metadata#',
'datum': 'http://unis.crest.iu.edu/schema/20151104/datum#',
'data': 'http://unis.crest.iu.edu/schema/20151104/data#',
'ipports': 'http://unis.crest.iu.edu/schema/ext/ipport/1/ipport#'
}
def __init__(self, rdf_file_name, host_name, port_number='8888'):
self.g=rdflib.Graph()
self.g.load(rdf_file_name)
self.prefix = """PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX ndl: <http://www.science.uva.nl/research/sne/ndl#> """;
self.uri="http://"+host_name+":"+port_number+"/"
print("URI:"+self.uri)
self.inode_object = IntermediateNode()
self.topology_object = Topology()
self.clean_all()
# self.decode()
def clean_all(self):
print("Starting the initial CLEANUP PROCESS")
self.clean_nodes()
self.delete_ports()
def clean_nodes(self):
print("Deleting the nodes")
nodes_uri=self.uri+"nodes"
nodes_list = coreapi.get(nodes_uri)
for node_dict in nodes_list:
print node_dict['selfRef']
# requests.delete(node_dict['selfRef'])
requests.delete(nodes_uri+"/"+node_dict['selfRef'].split("/")[3])
def delete_ports(self):
print("Deleting the ports")
ports_uri=self.uri+"ports"
ports_list = coreapi.get(ports_uri)
for port_dict in ports_list:
print port_dict['selfRef']
# requests.delete(port_dict['selfRef'])
requests.delete(ports_uri+"/"+port_dict['selfRef'].split("/")[3])
def decode(self):
print("Starting the DECODE PROCESS")
self.create_nodes_from_rdf()
self.update_node_refs()
self.create_ports_from_rdf()
self.update_port_refs()
self.build_port_ref_in_nodes()
print("Starting Links creation")
self.create_links_from_rdf()
print("Printing Inodes")
self.inode_object.print_inodes()
def create_nodes_from_rdf(self):
node_only_query=self.prefix+"""
SELECT ?name
WHERE {
?x rdf:type ndl:Device . ?x ndl:name ?name.
OPTIONAL {
?y ndl:connectedTo ?z .
?z rdf:type ndl:Interface .
?z ndl:name ?neighbour
} . OPTIONAL {
?y ndl:capacity ?capacity .
?y ndl:encapsulation ?type
} .
}"""
nodes_uri=self.uri+"nodes"
nodes = []
for row in self.g.query(node_only_query):
node = dict()
node["$schema"]=self.SCHEMAS['nodes']
node["name"]=row.name
# ports = []
# port={'href':'instageni.illinois.edu_authority_cm_slice_idms','rel': 'full'}
# ports.append(port)
# node["ports"]=ports
print "Node::"
print row.name
nodes.append(node)
print "::FINAL JSON::"
json_data = json.dumps(nodes)
print("NODE URI::"+nodes_uri)
print("JSON DATA:"+json_data)
requests.post(nodes_uri, data = json_data)
def update_node_refs(self):
nodes_uri=self.uri+"nodes"
nodes_list = coreapi.get(nodes_uri)
for check_node in nodes_list:
print check_node['name']
print check_node['selfRef']
node_object = Node(check_node['name'])
self.topology_object.add_node(self.getId(check_node['selfRef']), node_object)
print "\n"
print("PRINIIIIIII")
self.topology_object.display_topology()
############# PORTS
def getId(self, ref_url):
"""
http://10.10.0.135:8888/nodes/56f88569e1382308b0b6a2ea will return 56f88569e1382308b0b6a2ea
:param ref_url:
:return:
"""
return ref_url.split("/")[3]
def create_ref_url(self, type, id):
"""
It will build the url from type and id
:param type:
:param id:
:return:
"""
return self.uri+type+"/"+id
def create_ports_from_rdf(self):
interface_query=self.prefix+"""
SELECT ?name ?interface
WHERE {
?x rdf:type ndl:Device . ?x ndl:name ?name .
?x ndl:hasInterface ?interface
OPTIONAL {
?y ndl:connectedTo ?z .
?z rdf:type ndl:Interface .
?z ndl:name ?neighbour
} . OPTIONAL {
?y ndl:capacity ?capacity .
?y ndl:encapsulation ?type
} .
}"""
ports_uri=self.uri+"ports"
ports = []
print("PORTSSSSS!!!!")
for row in self.g.query(interface_query):
print(row.name+" :::"+row.interface)
temp_intf_name=row.interface
intf_name=temp_intf_name.split("#")
port_name_split=intf_name[1].split(":")
port_node_name=port_name_split[0]
port_name=port_name_split[1]
port = dict()
port["$schema"]=self.SCHEMAS['ports']
port["name"]=port_name
port["nodeRef"]=self.topology_object.get_node_id_by_name(port_node_name)
ipv4_addr = dict()
ipv4_addr["type"]="ipv4"
## TODO: Fill the actual IPv4 address probably from the interface table using Router Proxy
ipv4_addr["address"]="1.1.1.1"
port["properties"]={"ipv4":ipv4_addr}
ports.append(port)
print "::FINAL PORTS JSON::"
ports_json_data = json.dumps(ports)
print(ports_json_data)
print("PORTS URI::"+ports_uri)
requests.post(ports_uri, data = ports_json_data)
###### GET PORT REFS
def update_port_refs(self):
ports_uri=self.uri+"ports"
ports_list = coreapi.get(ports_uri)
for check_port in ports_list:
port_name = check_port['name']
port_ref = self.getId(check_port['selfRef'])
node_ref = self.getId(check_port['nodeRef'])
port_object = Port(node_ref, "2.2.2.2")
self.topology_object.nodes[node_ref].add_port(port_ref, port_object)
print "\n"
print("PRINIIIIIII TOPO FINAL")
self.topology_object.display_topology()
######### BUILD PORT REF in NODES
def build_port_ref_in_nodes(self):
for node_id in self.topology_object.nodes.keys():
self.topology_object.nodes[node_id].ports
node = dict()
ports = []
for port_id in self.topology_object.nodes[node_id].ports.keys():
port={'href': self.create_ref_url("ports", port_id),'rel': 'full'}
ports.append(port)
node["ports"]=ports
print "::FINAL JSON::"
json_data = json.dumps(node)
print(json_data)
r=requests.put(node_id, data=json_data)
def create_links_from_rdf(self):
links_query = self.prefix+"""
SELECT ?name ?interface ?connectedTo ?type ?neighbour
WHERE {
?x rdf:type ndl:Device . ?x ndl:name ?name .
?x ndl:hasInterface ?y . ?y rdf:type ndl:Interface .
?y ndl:name ?interface . ?y ndl:connectedTo ?connectedTo .
OPTIONAL {
?y ndl:connectedTo ?z .
?z rdf:type ndl:Interface .
?z ndl:name ?neighbour
} . OPTIONAL {
?y ndl:capacity ?capacity .
?y ndl:encapsulation ?type
} .
}"""
for row in self.g.query(links_query):
print(row['name'], row['interface'], row['connectedTo'])
dest_node_name = row['connectedTo'].split("#")[1]
print("ref::"+row['connectedTo'].split("#")[1])
print("router...name::"+row['interface'].split(":")[0])
node_name = row['interface'].split(":")[0]
print("interface...name::"+row['interface'].split(":")[1])
intf_name = row['interface'].split(":")[1]
self.inode_object.add_link_to_inode(node_name, intf_name, dest_node_name)
def build_links(self):
link = dict()
link["directed"] = "false"
link["$schema"] = self.SCHEMAS['links']
link["name"] = "linkk"
