def query_network(self, uuid, search_string, max_edges):
myConst = CX_CONSTANTS
niceCx = NiceCXNetwork()
#uuid = '7246d8cf-c644-11e6-b48c-0660b7976219'
search_terms_dict = {k:1 for k in search_string.split(',')}
solr = pysolr.Solr(solr_url + uuid + '/', timeout=10)
try:
results = solr.search(search_string, rows=10000)
#search_terms_array = [int(n['id']) for n in results.docs]
search_terms_array = {int(n['id']):1 for n in results.docs}
if(not search_terms_array):
return {'message': 'No nodes found'}
print('starting nodes 1')
#===================
# METADATA
#===================
available_aspects = []
for ae in (o for o in self.stream_aspect(uuid, 'metaData')):
available_aspects.append(ae.get(CX_CONSTANTS.METADATA_NAME))
mde = MetaDataElement(json_obj=ae)
niceCx.add_metadata(mde)
#available_aspects = ['edges', 'nodes'] # TODO - remove this
opaque_aspects = set(available_aspects).difference(known_aspects_min)
print(opaque_aspects)
#===================
# NODES
#===================
if 'nodes' in available_aspects:
for ae in (o for o in self.stream_aspect(uuid, 'nodes')):
if search_terms_array.get(ae.get(CX_CONSTANTS.ID)):
add_this_node = NodeElement(cx_fragment=ae)
niceCx.create_node(add_this_node)
else:
raise Exception('Network does not contain any nodes. Cannot query')
print('starting edges 1')
#===================
# EDGES
#===================
edge_count = 0
added_edges = 0
start_time = time.time()
if 'edges' in available_aspects:
for ae in (o for o in self.stream_aspect(uuid, 'edges')):
if niceCx.nodes.get(ae.get(CX_CONSTANTS.EDGE_SOURCE_NODE_ID_OR_SUBNETWORK)) is not None or niceCx.nodes.get(ae.get(CX_CONSTANTS.EDGE_TARGET_NODE_ID)) is not None:
add_this_edge = EdgeElement(cx_fragment=ae)
niceCx.create_edge(add_this_edge)
added_edges += 1
if edge_count % 5000 == 0:
print(edge_count)
#if edge_count > 30000:
# break
if added_edges > max_edges:
raise StopIteration('Max edges reached')
edge_count += 1
else:
raise Exception('Network does not contain any nodes. Cannot query')
print('Response time (Edge search): ' + str(time.time() - start_time))
print('starting nodes 2')
#===================
# NODES
#===================
for ae in (o for o in self.stream_aspect(uuid, 'nodes')):
if niceCx.get_missing_nodes().get(ae.get(CX_CONSTANTS.ID)):
add_this_node = NodeElement(cx_fragment=ae)
niceCx.create_node(add_this_node)
#====================
# NETWORK ATTRIBUTES
#====================
if 'networkAttributes' in available_aspects:
for ae in (o for o in self.stream_aspect(uuid, 'networkAttributes')):
add_this_network_attribute = NetworkAttributesElement(cx_fragment=ae)
niceCx.add_network_attribute(add_this_network_attribute)
#===================
# NODE ATTRIBUTES
#===================
if 'nodeAttributes' in available_aspects:
for ae in (o for o in self.stream_aspect(uuid, 'nodeAttributes')):
if niceCx.nodes.get(ae.get(CX_CONSTANTS.PROPERTY_OF)):
add_this_node_att = NodeAttributesElement(json_obj=ae)
niceCx.add_node_attribute(add_this_node_att)
#===================
# EDGE ATTRIBUTES
#===================
if 'edgeAttributes' in available_aspects:
for ae in (o for o in self.stream_aspect(uuid, 'edgeAttributes')):
if niceCx.edges.get(ae.get(CX_CONSTANTS.PROPERTY_OF)):
add_this_edge_att = EdgeAttributesElement(json_obj=ae)
niceCx.set_edge_attribute()
#===================
# NODE CITATIONS
#===================
if 'nodeCitations' in available_aspects:
for ae in (o for o in self.stream_aspect(uuid, 'nodeCitations')):
for e_po in ae.get(CX_CONSTANTS.PROPERTY_OF):
if niceCx.get_nodes().get(e_po) is not None:
niceCx.add_node_citations_from_cx(ae)
#===================
# EDGE CITATIONS
#===================
ec_count = 0
if 'edgeCitations' in available_aspects:
for ae in (o for o in self.stream_aspect(uuid, 'edgeCitations')):
for e_po in ae.get(CX_CONSTANTS.PROPERTY_OF):
if niceCx.get_edges().get(e_po) is not None:
niceCx.add_edge_citations_from_cx(ae)
ec_count += 1
if ec_count % 500 == 0:
print(ec_count)
#===================
# CITATIONS
#===================
if 'citations' in available_aspects:
#======================================================
# FILTER CITATIONS IF THERE ARE EDGE OR NODE CITATIONS
# OTHERWISE ADD THEM ALL (NO-FILTER) -- TODO
#======================================================
for ae in (o for o in self.stream_aspect(uuid, 'citations')):
add_this_citation = CitationElement(cx_fragment=ae)
niceCx.add_citation(add_this_citation)
#===================
# OPAQUE ASPECTS
#===================
for oa in opaque_aspects:
objects = self.stream_aspect(uuid, oa)
obj_items = (o for o in objects)
for oa_item in obj_items:
aspect_element = AspectElement(oa_item, oa)
niceCx.add_opaque_aspect(aspect_element)
except SolrError as se:
if('404' in se.message):
ndex2.get_logger('SOLR').warning('Network not found ' + self.uuid + ' on ' + solr_url + ' server.')
raise Exception("Network not found (SOLR)")
else:
ndex2.get_logger('SOLR').warning('Network error ' + self.uuid + ' on ' + solr_url + ' server. ' + se.message)
raise Exception(se.message)
except StopIteration as si:
ndex2.get_logger('QUERY').warning("Found more than max edges. Raising exception")
raise StopIteration(si.message)
#nice_cx_json = niceCx.to_cx()
return niceCx
def create_nice_cx_from_networkx(G):
"""
Create a NiceCXNetwork based on a networkx graph. The resulting NiceCXNetwork
contains the nodes edges and their attributes from the networkx graph and also
preserves the graph 'pos' attribute as a CX cartesian coordinates aspect.
:param G: networkx graph
:type G: networkx graph
:return: NiceCXNetwork
:rtype: NiceCXNetwork
"""
if G is None:
raise Exception('Networkx input is empty')
my_nicecx = NiceCXNetwork()
if G.graph.get('name'):
my_nicecx.set_name(G.graph.get('name'))
else:
my_nicecx.set_name('created from networkx')
my_nicecx.add_metadata_stub('networkAttributes')
for n, d in G.nodes_iter(data=True):
# =============
# ADD NODES
# =============
if d and d.get('name'):
my_nicecx.create_node(id=n,
node_name=d.get('name'),
node_represents=d.get('name'))
else:
my_nicecx.create_node(id=n, node_name=n, node_represents=n)
# ======================
# ADD NODE ATTRIBUTES
# ======================
for k, v in d.items():
attr_type = None
if isinstance(v, float):
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif isinstance(v, int):
attr_type = ATTRIBUTE_DATA_TYPE.INTEGER
my_nicecx.set_node_attribute(n, k, v, type=attr_type)
index = 0
for u, v, d in G.edges_iter(data=True):
# =============
# ADD EDGES
# =============
my_nicecx.create_edge(id=index,
edge_source=u,
edge_target=v,
edge_interaction=d.get('interaction'))
# ==============================
# ADD EDGE ATTRIBUTES
# ==============================
for k, val in d.items():
if k != 'interaction':
attr_type = None
if isinstance(val, float):
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif isinstance(val, int):
attr_type = ATTRIBUTE_DATA_TYPE.INTEGER
my_nicecx.set_edge_attribute(index, k, val, type=attr_type)
index += 1
my_nicecx.add_metadata_stub('nodes')
my_nicecx.add_metadata_stub('edges')
if my_nicecx.nodeAttributes:
my_nicecx.add_metadata_stub('nodeAttributes')
if my_nicecx.edgeAttributes:
my_nicecx.add_metadata_stub('edgeAttributes')
if hasattr(G, 'pos'):
aspect = _create_cartesian_coordinates_aspect_from_networkx(G)
my_nicecx.add_opaque_aspect('cartesianLayout', aspect)
my_nicecx.add_metadata_stub('cartesianLayout')
return my_nicecx
def create_nice_cx_from_networkx(G):
"""
Constructor that uses a networkx graph to build niceCX
:param G: networkx graph
:type G: networkx graph
:return: none
:rtype: none
"""
my_nicecx = NiceCXNetwork()
if G.graph.get('name'):
my_nicecx.set_name(G.graph.get('name'))
else:
my_nicecx.set_name('created from networkx')
my_nicecx.add_metadata_stub('networkAttributes')
for n, d in G.nodes_iter(data=True):
#=============
# ADD NODES
#=============
if d and d.get('name'):
my_nicecx.create_node(id=n,
node_name=d.get('name'),
node_represents=d.get('name'))
else:
my_nicecx.create_node(id=n, node_name=n, node_represents=n)
#======================
# ADD NODE ATTRIBUTES
#======================
for k, v in d.items():
my_nicecx.set_node_attribute(n, k, v)
index = 0
for u, v, d in G.edges_iter(data=True):
#=============
# ADD EDGES
#=============
my_nicecx.create_edge(id=index,
edge_source=u,
edge_target=v,
edge_interaction=d.get('interaction'))
#==============================
# ADD EDGE ATTRIBUTES
#==============================
for k, v in d.items():
if k != 'interaction':
my_nicecx.set_edge_attribute(index, k, v)
index += 1
#Cartesian aspect
cartesian_aspect = []
#for
my_nicecx.add_metadata_stub('nodes')
my_nicecx.add_metadata_stub('edges')
if my_nicecx.nodeAttributes:
my_nicecx.add_metadata_stub('nodeAttributes')
if my_nicecx.edgeAttributes:
my_nicecx.add_metadata_stub('edgeAttributes')
if hasattr(G, 'pos'):
G_pos = create_cartesian_coordinates_aspect_from_networkx(G)
my_nicecx.set_opaque_aspect('cartesianLayout',
G_pos.get('cartesianLayout'))
my_nicecx.add_metadata_stub('cartesianLayout')
return my_nicecx
def create_nice_cx_from_pandas(df,
source_field=None,
target_field=None,
source_node_attr=[],
target_node_attr=[],
edge_attr=[],
edge_interaction=None):
"""
Create a NiceCXNetwork from a pandas dataframe in which each row
specifies one edge in the network.
If only the df argument is provided the dataframe is treated as 'SIF' format,
where the first two columns specify the source and target node ids of the edge
and all other columns are ignored. The edge interaction is defaulted to "interacts-with"
If both the source_field and target_field arguments are provided, the those and any other
arguments refer to headers in the dataframe, controlling the mapping of columns to
the attributes of nodes, and edges in the resulting NiceCXNetwork. If a header is not
mapped the corresponding column is ignored. If the edge_interaction is not specified it
defaults to "interacts-with"
:param df: pandas dataframe to process
:param source_field: header name specifying the name of the source node.
:param target_field: header name specifying the name of the target node.
:param source_node_attr: list of header names specifying attributes of the source node.
:param target_node_attr: list of header names specifying attributes of the target node.
:param edge_attr: list of header names specifying attributes of the edge.
:param edge_interaction: the relationship between the source node and the target node, defaulting to "interacts-with"
:return: NiceCXNetwork
"""
my_nicecx = NiceCXNetwork()
# ====================================================
# IF NODE FIELD NAME (SOURCE AND TARGET) IS PROVIDED
# THEN USE THOSE FIELDS OTHERWISE USE INDEX 0 & 1
# ====================================================
my_nicecx.set_name('Pandas Upload')
my_nicecx.add_metadata_stub('networkAttributes')
count = 0
if source_field and target_field:
for index, row in df.iterrows():
if count % 10000 == 0:
print(count)
count += 1
# =============
# ADD NODES
# =============
my_nicecx.create_node(id=row[source_field],
node_name=row[source_field],
node_represents=row[source_field])
my_nicecx.create_node(id=row[target_field],
node_name=row[target_field],
node_represents=row[target_field])
# =============
# ADD EDGES
# =============
if edge_interaction:
if row.get(edge_interaction):
my_nicecx.create_edge(
id=index,
edge_source=row[source_field],
edge_target=row[target_field],
edge_interaction=row[edge_interaction])
else:
my_nicecx.create_edge(id=index,
edge_source=row[source_field],
edge_target=row[target_field],
edge_interaction=edge_interaction)
else:
my_nicecx.create_edge(id=index,
edge_source=row[source_field],
edge_target=row[target_field],
edge_interaction='interacts-with')
# ==============================
# ADD SOURCE NODE ATTRIBUTES
# ==============================
for sp in source_node_attr:
attr_type = None
if type(row[sp]) is float and math.isnan(row[sp]):
row[sp] = ''
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif type(row[sp]) is float and math.isinf(row[sp]):
row[sp] = 'Inf'
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif type(row[sp]) is float:
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif isinstance(row[sp], int):
attr_type = ATTRIBUTE_DATA_TYPE.INTEGER
my_nicecx.set_node_attribute(row[source_field],
sp,
row[sp],
type=attr_type)
# ==============================
# ADD TARGET NODE ATTRIBUTES
# ==============================
for tp in target_node_attr:
attr_type = None
if type(row[tp]) is float and math.isnan(row[tp]):
row[tp] = ''
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif type(row[tp]) is float and math.isinf(row[tp]):
row[tp] = 'Inf'
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif type(row[tp]) is float:
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif isinstance(row[tp], int):
attr_type = ATTRIBUTE_DATA_TYPE.INTEGER
my_nicecx.set_node_attribute(row[target_field],
tp,
row[tp],
type=attr_type)
# ==============================
# ADD EDGE ATTRIBUTES
# ==============================
for ep in edge_attr:
attr_type = None
if type(row[ep]) is float and math.isnan(row[ep]):
row[ep] = ''
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif type(row[ep]) is float and math.isinf(row[ep]):
row[ep] = 'INFINITY'
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
my_nicecx.set_edge_attribute(index,
ep,
row[ep],
type=attr_type)
else:
for index, row in df.iterrows():
# =============
# ADD NODES
# =============
my_nicecx.create_node(id=row[0],
node_name=row[0],
node_represents=row[0])
my_nicecx.create_node(id=row[1],
node_name=row[1],
node_represents=row[1])
# =============
# ADD EDGES
# =============
if len(row) > 2:
my_nicecx.create_edge(id=index,
edge_source=row[0],
edge_target=row[1],
edge_interaction=row[2])
else:
my_nicecx.create_edge(id=index,
edge_source=row[0],
edge_target=row[1],
edge_interaction='interacts-with')
my_nicecx.add_metadata_stub('nodes')
my_nicecx.add_metadata_stub('edges')
if source_node_attr or target_node_attr:
my_nicecx.add_metadata_stub('nodeAttributes')
if edge_attr:
my_nicecx.add_metadata_stub('edgeAttributes')
return my_nicecx
def create_nice_cx_from_pandas(df,
source_field=None,
target_field=None,
source_node_attr=[],
target_node_attr=[],
edge_attr=[],
edge_interaction=None):
"""
Constructor that uses a pandas dataframe to build niceCX
:param df: dataframe
:type df: Pandas Dataframe
:param headers:
:type headers:
:return: none
:rtype: n/a
"""
my_nicecx = NiceCXNetwork()
#====================================================
# IF NODE FIELD NAME (SOURCE AND TARGET) IS PROVIDED
# THEN USE THOSE FIELDS OTHERWISE USE INDEX 0 & 1
#====================================================
my_nicecx.set_name('Pandas Upload')
my_nicecx.add_metadata_stub('networkAttributes')
count = 0
if source_field and target_field:
for index, row in df.iterrows():
if count % 10000 == 0:
print(count)
count += 1
#=============
# ADD NODES
#=============
my_nicecx.create_node(id=row[source_field],
node_name=row[source_field],
node_represents=row[source_field])
my_nicecx.create_node(id=row[target_field],
node_name=row[target_field],
node_represents=row[target_field])
#=============
# ADD EDGES
#=============
if edge_interaction:
if row.get(edge_interaction):
my_nicecx.create_edge(
id=index,
edge_source=row[source_field],
edge_target=row[target_field],
edge_interaction=row[edge_interaction])
else:
my_nicecx.create_edge(id=index,
edge_source=row[source_field],
edge_target=row[target_field],
edge_interaction=edge_interaction)
else:
my_nicecx.create_edge(id=index,
edge_source=row[source_field],
edge_target=row[target_field],
edge_interaction='neighbor-of')
#==============================
# ADD SOURCE NODE ATTRIBUTES
#==============================
for sp in source_node_attr:
attr_type = None
if type(row[sp]) is float and math.isnan(row[sp]):
row[sp] = ''
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif type(row[sp]) is float and math.isinf(row[sp]):
row[sp] = 'Inf'
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
my_nicecx.set_node_attribute(row[source_field],
sp,
row[sp],
type=attr_type)
#==============================
# ADD TARGET NODE ATTRIBUTES
#==============================
for tp in target_node_attr:
attr_type = None
if type(row[tp]) is float and math.isnan(row[tp]):
row[tp] = ''
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif type(row[tp]) is float and math.isinf(row[tp]):
row[tp] = 'Inf'
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
my_nicecx.set_node_attribute(row[target_field],
tp,
row[tp],
type=attr_type)
#==============================
# ADD EDGE ATTRIBUTES
#==============================
for ep in edge_attr:
attr_type = None
if type(row[ep]) is float and math.isnan(row[ep]):
row[ep] = ''
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
elif type(row[ep]) is float and math.isinf(row[ep]):
row[ep] = 'INFINITY'
attr_type = ATTRIBUTE_DATA_TYPE.FLOAT
my_nicecx.set_edge_attribute(index,
ep,
row[ep],
type=attr_type)
else:
for index, row in df.iterrows():
#=============
# ADD NODES
#=============
my_nicecx.create_node(id=row[0],
node_name=row[0],
node_represents=row[0])
my_nicecx.create_node(id=row[1],
node_name=row[1],
node_represents=row[1])
#=============
# ADD EDGES
#=============
if len(row) > 2:
my_nicecx.create_edge(id=index,
edge_source=row[0],
edge_target=row[1],
edge_interaction=row[2])
else:
my_nicecx.create_edge(id=index,
edge_source=row[0],
edge_target=row[1],
edge_interaction='interacts-with')
my_nicecx.add_metadata_stub('nodes')
my_nicecx.add_metadata_stub('edges')
if source_node_attr or target_node_attr:
my_nicecx.add_metadata_stub('nodeAttributes')
if edge_attr:
my_nicecx.add_metadata_stub('edgeAttributes')
return my_nicecx