def create_pop(pop_url, uuid, timestamp, properties=None):
"""
Create a node in PoP DB representing a Point Of Presence
:param pop_url: Url of PoP DB
:param uuid: Uuid of the node to be created
:param timestamp: timestamp of the creation
:param properties: optionally dict containing key values representing
PoP Properties
"""
if not properties:
properties = dict()
properties['type'] = 'pop'
index = ('pop', 'uuid', uuid)
graph_db = neo4j.Graph(pop_url)
neo_resource.add_node(graph_db, index, timestamp, properties=properties)
def create_pop(pop_url, uuid, timestamp, properties=None):
"""
Create a node in PoP DB representing a Point Of Presence
:param pop_url: Url of PoP DB
:param uuid: Uuid of the node to be created
:param timestamp: timestamp of the creation
:param properties: optionally dict containing key values representing
PoP Properties
"""
if not properties:
properties = dict()
properties['type'] = 'pop'
index = ('pop', 'uuid', uuid)
graph_db = neo4j.Graph(pop_url)
neo_resource.add_node(graph_db, index, timestamp, properties=properties)
def get_or_add_hypervisor(self, graph_db, resource_id, pop, timestamp, properties=None):
"""
Retrieve or add the Hypervisor node from the DB
:param graph_db: Graph DB instance
:param resource_id: ID of the Hypervisor
:param pop: PoP ID
:param timestamp: timestamp in epoch
:param properties: optional properties of the node
:return Node: Hypervisor node
"""
if not properties:
properties = dict()
with mutex:
properties['pop'] = pop
index = (self.label, 'openstack_uuid', 'hypervisor-' + str(resource_id))
hyperv_node = neo_resource.add_node(graph_db,
index,
timestamp,
properties)
if self.hostname:
host_node = HostNode(self.hostname).get_resource(graph_db, timestamp)
if host_node:
neo_resource.add_edge(graph_db, hyperv_node, host_node, timestamp, 'runs_on')
return hyperv_node
def get_or_add_hypervisor(self,
graph_db,
resource_id,
pop,
timestamp,
properties=None):
"""
Retrieve or add the Hypervisor node from the DB
:param graph_db: Graph DB instance
:param resource_id: ID of the Hypervisor
:param pop: PoP ID
:param timestamp: timestamp in epoch
:param properties: optional properties of the node
:return Node: Hypervisor node
"""
if not properties:
properties = dict()
with mutex:
properties['pop'] = pop
index = (self.label, 'openstack_uuid',
'hypervisor-' + str(resource_id))
hyperv_node = neo_resource.add_node(graph_db, index, timestamp,
properties)
if self.hostname:
host_node = HostNode(self.hostname).get_resource(
graph_db, timestamp)
if host_node:
neo_resource.add_edge(graph_db, hyperv_node, host_node,
timestamp, 'runs_on')
return hyperv_node
def get_or_add_resource(self, graph_db, timestamp, properties=None):
"""
Retrieve or add the resource
:param graph_db: Graph DB instance
:param timestamp: timestamp in epoch
:param properties: optional properties of the node
:return node: Resource node
"""
node = neo_resource.add_node(graph_db, self.index, timestamp, properties=properties)
return node
def get_resource(self, graph_db, timestamp):
"""
Retrieve or add machine node
:param graph_db: Graph DB instance
:param timestamp: timestamp in epoch
:return Node: Machine node
"""
node_name = self.hostname + '_' + 'Machine' + '_0'
index = ('physical_resource', 'physical_name', node_name)
node = neo_resource.add_node(graph_db, index, timestamp)
return node
def get_resource(self, graph_db, timestamp):
"""
Retrieve or add machine node
:param graph_db: Graph DB instance
:param timestamp: timestamp in epoch
:return Node: Machine node
"""
node_name = self.hostname + '_' + 'Machine' + '_0'
index = ('physical_resource', 'physical_name', node_name)
node = neo_resource.add_node(graph_db, index, timestamp)
return node
def get_or_add_resource(self, graph_db, timestamp, properties=None):
"""
Retrieve or add the resource
:param graph_db: Graph DB instance
:param timestamp: timestamp in epoch
:param properties: optional properties of the node
:return node: Resource node
"""
node = neo_resource.add_node(graph_db,
self.index,
timestamp,
properties=properties)
return node
def store(self, path, hwloc_file, cpu_file=None, sriov_file=None, dpdk_file=None, timestamp=None):
"""
Store information contained in files created by the EPA agents into Neo4j.
using a networkx graph
:param path: Path of the files
:param hwloc_file: Hardware locality file
:param cpu_file: Optional cpu information file
:param sriov_file: Optional SR-IOV information file
:param dpdk_file: Optional DPDK information file
:param timestamp: Optional timestamp in epoch
"""
graph = nx.DiGraph()
xml_root = Et.parse(path + hwloc_file).getroot()
deleted_edges = {}
for child in xml_root:
_parse_object_hwloc(graph, child, self.hostname, deleted_edges, self.pop_id)
if cpu_file is not None:
processors_dict = _parse_cpu_info(path + cpu_file)
_enrich_graph_cpuinfo(graph, processors_dict)
if dpdk_file is not None:
dpdk_dict = _parse_dpdk_info(path + dpdk_file)
_enrich_graph_dpdkinfo(graph, dpdk_dict)
if sriov_file is not None:
sriov_dict = _parse_sriov_info(path + sriov_file)
_enrich_graph_sriovinfo(graph, sriov_dict)
if timestamp is not None:
now = timestamp
else:
now = time.time()
neo_id_nodes = {}
nodes_to_add = []
nodes_stored = []
query_string = (
"Match n Where n.hostname = {hostname} " "And n.resource_type = {resource_type} Return n.physical_name"
)
res = self.graph_db.cypher.execute(query_string, hostname=self.hostname, resource_type="physical")
for item in res:
print str(item)
nodes_stored.append(item["n.physical_name"])
for nx_node in graph.nodes():
nodes_to_add.append(str(nx_node))
neo_id_nodes[nx_node] = neo_resource.add_node(
self.graph_db, (self.label, self.index, nx_node), now, get_node_properties(graph, nx_node)
)
nodes_to_remove = [item for item in nodes_stored if item not in nodes_to_add]
for node in nodes_to_remove:
neo_resource.delete_node(self.graph_db, (self.label, self.index, node))
for edge in graph.edges():
source = edge[0]
target = edge[1]
edge_label = ""
if "label" in graph.edge[source][target]:
edge_label = graph.edge[source][target]["label"]
db_src = neo_id_nodes[source]
db_target = neo_id_nodes[target]
rel_stored = neo_resource.get_edge(self.graph_db, db_src, db_target)
if rel_stored is None:
neo_resource.add_edge(self.graph_db, db_src, db_target, timestamp, edge_label)
else:
neo_resource.update_edge(self.graph_db, timestamp, edge_label, db_src=db_src, db_target=db_target)
def store(self, graph_db, properties, pop, timestamp,
in_edges=None, out_edges=None, host_nodes=None,
controller_services=None, hypervisors=None, update=False):
"""
Store the resource to the Graph DB
:param graph_db: Graph DB instance
:param properties: properties of the node
:param pop: PoP ID
:param timestamp: timestamp in epoch
:param in_edges: inward relations
:param out_edges: outward relations
:param host_nodes: Machine node where the resource is running on
:param controller_services: Controller services that manage the resource
:param hypervisors: hypervisor where the resource is running on, if any
:param update: if true, it updates an existing node
"""
with mutex:
properties['pop'] = pop
if update:
openstack_node = neo_resource.update_node(graph_db, self.index, timestamp, properties=properties)
else:
openstack_node = neo_resource.add_node(graph_db, self.index, timestamp, properties=properties)
if openstack_node:
if in_edges:
for in_edge in in_edges:
if in_edges[in_edge]['mandatory']:
in_node = OpenstackResource(in_edge).get_or_add_resource(graph_db, timestamp)
else:
in_node = OpenstackResource(in_edge).get_resource(graph_db)
if in_node:
neo_resource.add_edge(graph_db, in_node, openstack_node, timestamp,
in_edges[in_edge]['label'])
if out_edges:
for out_edge in out_edges:
if out_edges[out_edge]['mandatory']:
out_node = OpenstackResource(out_edge).get_or_add_resource(graph_db, timestamp)
else:
out_node = OpenstackResource(out_edge).get_resource(graph_db)
if out_node:
neo_resource.add_edge(graph_db, openstack_node, out_node, timestamp,
out_edges[out_edge]['label'])
if host_nodes:
for host in host_nodes:
host_node = HostNode(host).get_resource(graph_db, timestamp)
if host_node:
neo_resource.add_edge(graph_db, openstack_node, host_node, timestamp, 'runs_on')
if hypervisors:
for host in hypervisors:
hypervisor = Hypervisor(host).get_hypervisor(graph_db)
if hypervisor:
neo_resource.add_edge(graph_db, openstack_node, hypervisor, timestamp, 'deployed_on')
if controller_services:
for service in controller_services:
service_node = ControllerService().get_resource_by_type(graph_db, service)
if service_node:
neo_resource.add_edge(graph_db, service_node, openstack_node,
timestamp, 'manages')
if openstack_node.properties['type'] == 'port':
if 'profile' in openstack_node.properties['attributes']:
port_attributes = json.loads(openstack_node.properties['attributes'])
if 'profile' in port_attributes and 'pci_slot' in port_attributes['profile']:
pci_dev = neo_resource.get_nodes_by_attribute(graph_db,
openstack_node.properties['hostname'],
port_attributes['profile']['pci_slot'])
if pci_dev and len(pci_dev) == 1:
neo_resource.add_edge(graph_db, openstack_node, pci_dev[0],
timestamp, 'runs_on')
return openstack_node
def store(self,
graph_db,
properties,
pop,
timestamp,
in_edges=None,
out_edges=None,
host_nodes=None,
controller_services=None,
hypervisors=None,
update=False):
"""
Store the resource to the Graph DB
:param graph_db: Graph DB instance
:param properties: properties of the node
:param pop: PoP ID
:param timestamp: timestamp in epoch
:param in_edges: inward relations
:param out_edges: outward relations
:param host_nodes: Machine node where the resource is running on
:param controller_services: Controller services that manage the resource
:param hypervisors: hypervisor where the resource is running on, if any
:param update: if true, it updates an existing node
"""
with mutex:
properties['pop'] = pop
if update:
openstack_node = neo_resource.update_node(
graph_db, self.index, timestamp, properties=properties)
else:
openstack_node = neo_resource.add_node(graph_db,
self.index,
timestamp,
properties=properties)
if openstack_node:
if in_edges:
for in_edge in in_edges:
if in_edges[in_edge]['mandatory']:
in_node = OpenstackResource(
in_edge).get_or_add_resource(
graph_db, timestamp)
else:
in_node = OpenstackResource(in_edge).get_resource(
graph_db)
if in_node:
neo_resource.add_edge(graph_db, in_node,
openstack_node, timestamp,
in_edges[in_edge]['label'])
if out_edges:
for out_edge in out_edges:
if out_edges[out_edge]['mandatory']:
out_node = OpenstackResource(
out_edge).get_or_add_resource(
graph_db, timestamp)
else:
out_node = OpenstackResource(
out_edge).get_resource(graph_db)
if out_node:
neo_resource.add_edge(graph_db, openstack_node,
out_node, timestamp,
out_edges[out_edge]['label'])
if host_nodes:
for host in host_nodes:
host_node = HostNode(host).get_resource(
graph_db, timestamp)
if host_node:
neo_resource.add_edge(graph_db, openstack_node,
host_node, timestamp,
'runs_on')
if hypervisors:
for host in hypervisors:
hypervisor = Hypervisor(host).get_hypervisor(graph_db)
if hypervisor:
neo_resource.add_edge(graph_db, openstack_node,
hypervisor, timestamp,
'deployed_on')
if controller_services:
for service in controller_services:
service_node = ControllerService(
).get_resource_by_type(graph_db, service)
if service_node:
neo_resource.add_edge(graph_db, service_node,
openstack_node, timestamp,
'manages')
if openstack_node.properties['type'] == 'port':
if 'profile' in openstack_node.properties['attributes']:
port_attributes = json.loads(
openstack_node.properties['attributes'])
if 'profile' in port_attributes and 'pci_slot' in port_attributes[
'profile']:
pci_dev = neo_resource.get_nodes_by_attribute(
graph_db,
openstack_node.properties['hostname'],
port_attributes['profile']['pci_slot'])
if pci_dev and len(pci_dev) == 1:
neo_resource.add_edge(graph_db, openstack_node,
pci_dev[0], timestamp,
'runs_on')
return openstack_node
def store(self,
path,
hwloc_file,
cpu_file=None,
sriov_file=None,
dpdk_file=None,
timestamp=None):
"""
Store information contained in files created by the EPA agents into Neo4j.
using a networkx graph
:param path: Path of the files
:param hwloc_file: Hardware locality file
:param cpu_file: Optional cpu information file
:param sriov_file: Optional SR-IOV information file
:param dpdk_file: Optional DPDK information file
:param timestamp: Optional timestamp in epoch
"""
graph = nx.DiGraph()
xml_root = Et.parse(path + hwloc_file).getroot()
deleted_edges = {}
for child in xml_root:
_parse_object_hwloc(graph, child, self.hostname, deleted_edges,
self.pop_id)
if cpu_file is not None:
processors_dict = _parse_cpu_info(path + cpu_file)
_enrich_graph_cpuinfo(graph, processors_dict)
if dpdk_file is not None:
dpdk_dict = _parse_dpdk_info(path + dpdk_file)
_enrich_graph_dpdkinfo(graph, dpdk_dict)
if sriov_file is not None:
sriov_dict = _parse_sriov_info(path + sriov_file)
_enrich_graph_sriovinfo(graph, sriov_dict)
if timestamp is not None:
now = timestamp
else:
now = time.time()
neo_id_nodes = {}
nodes_to_add = []
nodes_stored = []
query_string = 'Match n Where n.hostname = {hostname} ' \
'And n.resource_type = {resource_type} Return n.physical_name'
res = self.graph_db.cypher.execute(query_string,
hostname=self.hostname,
resource_type='physical')
for item in res:
print str(item)
nodes_stored.append(item['n.physical_name'])
for nx_node in graph.nodes():
nodes_to_add.append(str(nx_node))
neo_id_nodes[nx_node] = neo_resource.add_node(
self.graph_db, (self.label, self.index, nx_node), now,
get_node_properties(graph, nx_node))
nodes_to_remove = [
item for item in nodes_stored if item not in nodes_to_add
]
for node in nodes_to_remove:
neo_resource.delete_node(self.graph_db,
(self.label, self.index, node))
for edge in graph.edges():
source = edge[0]
target = edge[1]
edge_label = ''
if 'label' in graph.edge[source][target]:
edge_label = graph.edge[source][target]['label']
db_src = neo_id_nodes[source]
db_target = neo_id_nodes[target]
rel_stored = neo_resource.get_edge(self.graph_db, db_src,
db_target)
if rel_stored is None:
neo_resource.add_edge(self.graph_db, db_src, db_target,
timestamp, edge_label)
else:
neo_resource.update_edge(self.graph_db,
timestamp,
edge_label,
db_src=db_src,
db_target=db_target)