def query_network(self, uuid, search_string, max_edges): myConst = CX_CONSTANTS niceCx = NiceCXNetwork() #uuid = '7246d8cf-c644-11e6-b48c-0660b7976219' search_terms_dict = {k:1 for k in search_string.split(',')} solr = pysolr.Solr(solr_url + uuid + '/', timeout=10) try: results = solr.search(search_string, rows=10000) #search_terms_array = [int(n['id']) for n in results.docs] search_terms_array = {int(n['id']):1 for n in results.docs} if(not search_terms_array): return {'message': 'No nodes found'} print('starting nodes 1') #=================== # METADATA #=================== available_aspects = [] for ae in (o for o in self.stream_aspect(uuid, 'metaData')): available_aspects.append(ae.get(CX_CONSTANTS.METADATA_NAME)) mde = MetaDataElement(json_obj=ae) niceCx.add_metadata(mde) #available_aspects = ['edges', 'nodes'] # TODO - remove this opaque_aspects = set(available_aspects).difference(known_aspects_min) print(opaque_aspects) #=================== # NODES #=================== if 'nodes' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'nodes')): if search_terms_array.get(ae.get(CX_CONSTANTS.ID)): add_this_node = NodeElement(cx_fragment=ae) niceCx.create_node(add_this_node) else: raise Exception('Network does not contain any nodes. Cannot query') print('starting edges 1') #=================== # EDGES #=================== edge_count = 0 added_edges = 0 start_time = time.time() if 'edges' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'edges')): if niceCx.nodes.get(ae.get(CX_CONSTANTS.EDGE_SOURCE_NODE_ID_OR_SUBNETWORK)) is not None or niceCx.nodes.get(ae.get(CX_CONSTANTS.EDGE_TARGET_NODE_ID)) is not None: add_this_edge = EdgeElement(cx_fragment=ae) niceCx.create_edge(add_this_edge) added_edges += 1 if edge_count % 5000 == 0: print(edge_count) #if edge_count > 30000: # break if added_edges > max_edges: raise StopIteration('Max edges reached') edge_count += 1 else: raise Exception('Network does not contain any nodes. Cannot query') print('Response time (Edge search): ' + str(time.time() - start_time)) print('starting nodes 2') #=================== # NODES #=================== for ae in (o for o in self.stream_aspect(uuid, 'nodes')): if niceCx.get_missing_nodes().get(ae.get(CX_CONSTANTS.ID)): add_this_node = NodeElement(cx_fragment=ae) niceCx.create_node(add_this_node) #==================== # NETWORK ATTRIBUTES #==================== if 'networkAttributes' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'networkAttributes')): add_this_network_attribute = NetworkAttributesElement(cx_fragment=ae) niceCx.add_network_attribute(add_this_network_attribute) #=================== # NODE ATTRIBUTES #=================== if 'nodeAttributes' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'nodeAttributes')): if niceCx.nodes.get(ae.get(CX_CONSTANTS.PROPERTY_OF)): add_this_node_att = NodeAttributesElement(json_obj=ae) niceCx.add_node_attribute(add_this_node_att) #=================== # EDGE ATTRIBUTES #=================== if 'edgeAttributes' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'edgeAttributes')): if niceCx.edges.get(ae.get(CX_CONSTANTS.PROPERTY_OF)): add_this_edge_att = EdgeAttributesElement(json_obj=ae) niceCx.set_edge_attribute() #=================== # NODE CITATIONS #=================== if 'nodeCitations' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'nodeCitations')): for e_po in ae.get(CX_CONSTANTS.PROPERTY_OF): if niceCx.get_nodes().get(e_po) is not None: niceCx.add_node_citations_from_cx(ae) #=================== # EDGE CITATIONS #=================== ec_count = 0 if 'edgeCitations' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'edgeCitations')): for e_po in ae.get(CX_CONSTANTS.PROPERTY_OF): if niceCx.get_edges().get(e_po) is not None: niceCx.add_edge_citations_from_cx(ae) ec_count += 1 if ec_count % 500 == 0: print(ec_count) #=================== # CITATIONS #=================== if 'citations' in available_aspects: #====================================================== # FILTER CITATIONS IF THERE ARE EDGE OR NODE CITATIONS # OTHERWISE ADD THEM ALL (NO-FILTER) -- TODO #====================================================== for ae in (o for o in self.stream_aspect(uuid, 'citations')): add_this_citation = CitationElement(cx_fragment=ae) niceCx.add_citation(add_this_citation) #=================== # OPAQUE ASPECTS #=================== for oa in opaque_aspects: objects = self.stream_aspect(uuid, oa) obj_items = (o for o in objects) for oa_item in obj_items: aspect_element = AspectElement(oa_item, oa) niceCx.add_opaque_aspect(aspect_element) except SolrError as se: if('404' in se.message): ndex2.get_logger('SOLR').warning('Network not found ' + self.uuid + ' on ' + solr_url + ' server.') raise Exception("Network not found (SOLR)") else: ndex2.get_logger('SOLR').warning('Network error ' + self.uuid + ' on ' + solr_url + ' server. ' + se.message) raise Exception(se.message) except StopIteration as si: ndex2.get_logger('QUERY').warning("Found more than max edges. Raising exception") raise StopIteration(si.message) #nice_cx_json = niceCx.to_cx() return niceCx
def create_nice_cx_from_networkx(G): """ Create a NiceCXNetwork based on a networkx graph. The resulting NiceCXNetwork contains the nodes edges and their attributes from the networkx graph and also preserves the graph 'pos' attribute as a CX cartesian coordinates aspect. :param G: networkx graph :type G: networkx graph :return: NiceCXNetwork :rtype: NiceCXNetwork """ if G is None: raise Exception('Networkx input is empty') my_nicecx = NiceCXNetwork() if G.graph.get('name'): my_nicecx.set_name(G.graph.get('name')) else: my_nicecx.set_name('created from networkx') my_nicecx.add_metadata_stub('networkAttributes') for n, d in G.nodes_iter(data=True): # ============= # ADD NODES # ============= if d and d.get('name'): my_nicecx.create_node(id=n, node_name=d.get('name'), node_represents=d.get('name')) else: my_nicecx.create_node(id=n, node_name=n, node_represents=n) # ====================== # ADD NODE ATTRIBUTES # ====================== for k, v in d.items(): attr_type = None if isinstance(v, float): attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif isinstance(v, int): attr_type = ATTRIBUTE_DATA_TYPE.INTEGER my_nicecx.set_node_attribute(n, k, v, type=attr_type) index = 0 for u, v, d in G.edges_iter(data=True): # ============= # ADD EDGES # ============= my_nicecx.create_edge(id=index, edge_source=u, edge_target=v, edge_interaction=d.get('interaction')) # ============================== # ADD EDGE ATTRIBUTES # ============================== for k, val in d.items(): if k != 'interaction': attr_type = None if isinstance(val, float): attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif isinstance(val, int): attr_type = ATTRIBUTE_DATA_TYPE.INTEGER my_nicecx.set_edge_attribute(index, k, val, type=attr_type) index += 1 my_nicecx.add_metadata_stub('nodes') my_nicecx.add_metadata_stub('edges') if my_nicecx.nodeAttributes: my_nicecx.add_metadata_stub('nodeAttributes') if my_nicecx.edgeAttributes: my_nicecx.add_metadata_stub('edgeAttributes') if hasattr(G, 'pos'): aspect = _create_cartesian_coordinates_aspect_from_networkx(G) my_nicecx.add_opaque_aspect('cartesianLayout', aspect) my_nicecx.add_metadata_stub('cartesianLayout') return my_nicecx
def create_nice_cx_from_networkx(G): """ Constructor that uses a networkx graph to build niceCX :param G: networkx graph :type G: networkx graph :return: none :rtype: none """ my_nicecx = NiceCXNetwork() if G.graph.get('name'): my_nicecx.set_name(G.graph.get('name')) else: my_nicecx.set_name('created from networkx') my_nicecx.add_metadata_stub('networkAttributes') for n, d in G.nodes_iter(data=True): #============= # ADD NODES #============= if d and d.get('name'): my_nicecx.create_node(id=n, node_name=d.get('name'), node_represents=d.get('name')) else: my_nicecx.create_node(id=n, node_name=n, node_represents=n) #====================== # ADD NODE ATTRIBUTES #====================== for k, v in d.items(): my_nicecx.set_node_attribute(n, k, v) index = 0 for u, v, d in G.edges_iter(data=True): #============= # ADD EDGES #============= my_nicecx.create_edge(id=index, edge_source=u, edge_target=v, edge_interaction=d.get('interaction')) #============================== # ADD EDGE ATTRIBUTES #============================== for k, v in d.items(): if k != 'interaction': my_nicecx.set_edge_attribute(index, k, v) index += 1 #Cartesian aspect cartesian_aspect = [] #for my_nicecx.add_metadata_stub('nodes') my_nicecx.add_metadata_stub('edges') if my_nicecx.nodeAttributes: my_nicecx.add_metadata_stub('nodeAttributes') if my_nicecx.edgeAttributes: my_nicecx.add_metadata_stub('edgeAttributes') if hasattr(G, 'pos'): G_pos = create_cartesian_coordinates_aspect_from_networkx(G) my_nicecx.set_opaque_aspect('cartesianLayout', G_pos.get('cartesianLayout')) my_nicecx.add_metadata_stub('cartesianLayout') return my_nicecx
def create_nice_cx_from_pandas(df, source_field=None, target_field=None, source_node_attr=[], target_node_attr=[], edge_attr=[], edge_interaction=None): """ Create a NiceCXNetwork from a pandas dataframe in which each row specifies one edge in the network. If only the df argument is provided the dataframe is treated as 'SIF' format, where the first two columns specify the source and target node ids of the edge and all other columns are ignored. The edge interaction is defaulted to "interacts-with" If both the source_field and target_field arguments are provided, the those and any other arguments refer to headers in the dataframe, controlling the mapping of columns to the attributes of nodes, and edges in the resulting NiceCXNetwork. If a header is not mapped the corresponding column is ignored. If the edge_interaction is not specified it defaults to "interacts-with" :param df: pandas dataframe to process :param source_field: header name specifying the name of the source node. :param target_field: header name specifying the name of the target node. :param source_node_attr: list of header names specifying attributes of the source node. :param target_node_attr: list of header names specifying attributes of the target node. :param edge_attr: list of header names specifying attributes of the edge. :param edge_interaction: the relationship between the source node and the target node, defaulting to "interacts-with" :return: NiceCXNetwork """ my_nicecx = NiceCXNetwork() # ==================================================== # IF NODE FIELD NAME (SOURCE AND TARGET) IS PROVIDED # THEN USE THOSE FIELDS OTHERWISE USE INDEX 0 & 1 # ==================================================== my_nicecx.set_name('Pandas Upload') my_nicecx.add_metadata_stub('networkAttributes') count = 0 if source_field and target_field: for index, row in df.iterrows(): if count % 10000 == 0: print(count) count += 1 # ============= # ADD NODES # ============= my_nicecx.create_node(id=row[source_field], node_name=row[source_field], node_represents=row[source_field]) my_nicecx.create_node(id=row[target_field], node_name=row[target_field], node_represents=row[target_field]) # ============= # ADD EDGES # ============= if edge_interaction: if row.get(edge_interaction): my_nicecx.create_edge( id=index, edge_source=row[source_field], edge_target=row[target_field], edge_interaction=row[edge_interaction]) else: my_nicecx.create_edge(id=index, edge_source=row[source_field], edge_target=row[target_field], edge_interaction=edge_interaction) else: my_nicecx.create_edge(id=index, edge_source=row[source_field], edge_target=row[target_field], edge_interaction='interacts-with') # ============================== # ADD SOURCE NODE ATTRIBUTES # ============================== for sp in source_node_attr: attr_type = None if type(row[sp]) is float and math.isnan(row[sp]): row[sp] = '' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif type(row[sp]) is float and math.isinf(row[sp]): row[sp] = 'Inf' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif type(row[sp]) is float: attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif isinstance(row[sp], int): attr_type = ATTRIBUTE_DATA_TYPE.INTEGER my_nicecx.set_node_attribute(row[source_field], sp, row[sp], type=attr_type) # ============================== # ADD TARGET NODE ATTRIBUTES # ============================== for tp in target_node_attr: attr_type = None if type(row[tp]) is float and math.isnan(row[tp]): row[tp] = '' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif type(row[tp]) is float and math.isinf(row[tp]): row[tp] = 'Inf' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif type(row[tp]) is float: attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif isinstance(row[tp], int): attr_type = ATTRIBUTE_DATA_TYPE.INTEGER my_nicecx.set_node_attribute(row[target_field], tp, row[tp], type=attr_type) # ============================== # ADD EDGE ATTRIBUTES # ============================== for ep in edge_attr: attr_type = None if type(row[ep]) is float and math.isnan(row[ep]): row[ep] = '' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif type(row[ep]) is float and math.isinf(row[ep]): row[ep] = 'INFINITY' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT my_nicecx.set_edge_attribute(index, ep, row[ep], type=attr_type) else: for index, row in df.iterrows(): # ============= # ADD NODES # ============= my_nicecx.create_node(id=row[0], node_name=row[0], node_represents=row[0]) my_nicecx.create_node(id=row[1], node_name=row[1], node_represents=row[1]) # ============= # ADD EDGES # ============= if len(row) > 2: my_nicecx.create_edge(id=index, edge_source=row[0], edge_target=row[1], edge_interaction=row[2]) else: my_nicecx.create_edge(id=index, edge_source=row[0], edge_target=row[1], edge_interaction='interacts-with') my_nicecx.add_metadata_stub('nodes') my_nicecx.add_metadata_stub('edges') if source_node_attr or target_node_attr: my_nicecx.add_metadata_stub('nodeAttributes') if edge_attr: my_nicecx.add_metadata_stub('edgeAttributes') return my_nicecx
def create_nice_cx_from_pandas(df, source_field=None, target_field=None, source_node_attr=[], target_node_attr=[], edge_attr=[], edge_interaction=None): """ Constructor that uses a pandas dataframe to build niceCX :param df: dataframe :type df: Pandas Dataframe :param headers: :type headers: :return: none :rtype: n/a """ my_nicecx = NiceCXNetwork() #==================================================== # IF NODE FIELD NAME (SOURCE AND TARGET) IS PROVIDED # THEN USE THOSE FIELDS OTHERWISE USE INDEX 0 & 1 #==================================================== my_nicecx.set_name('Pandas Upload') my_nicecx.add_metadata_stub('networkAttributes') count = 0 if source_field and target_field: for index, row in df.iterrows(): if count % 10000 == 0: print(count) count += 1 #============= # ADD NODES #============= my_nicecx.create_node(id=row[source_field], node_name=row[source_field], node_represents=row[source_field]) my_nicecx.create_node(id=row[target_field], node_name=row[target_field], node_represents=row[target_field]) #============= # ADD EDGES #============= if edge_interaction: if row.get(edge_interaction): my_nicecx.create_edge( id=index, edge_source=row[source_field], edge_target=row[target_field], edge_interaction=row[edge_interaction]) else: my_nicecx.create_edge(id=index, edge_source=row[source_field], edge_target=row[target_field], edge_interaction=edge_interaction) else: my_nicecx.create_edge(id=index, edge_source=row[source_field], edge_target=row[target_field], edge_interaction='neighbor-of') #============================== # ADD SOURCE NODE ATTRIBUTES #============================== for sp in source_node_attr: attr_type = None if type(row[sp]) is float and math.isnan(row[sp]): row[sp] = '' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif type(row[sp]) is float and math.isinf(row[sp]): row[sp] = 'Inf' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT my_nicecx.set_node_attribute(row[source_field], sp, row[sp], type=attr_type) #============================== # ADD TARGET NODE ATTRIBUTES #============================== for tp in target_node_attr: attr_type = None if type(row[tp]) is float and math.isnan(row[tp]): row[tp] = '' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif type(row[tp]) is float and math.isinf(row[tp]): row[tp] = 'Inf' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT my_nicecx.set_node_attribute(row[target_field], tp, row[tp], type=attr_type) #============================== # ADD EDGE ATTRIBUTES #============================== for ep in edge_attr: attr_type = None if type(row[ep]) is float and math.isnan(row[ep]): row[ep] = '' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT elif type(row[ep]) is float and math.isinf(row[ep]): row[ep] = 'INFINITY' attr_type = ATTRIBUTE_DATA_TYPE.FLOAT my_nicecx.set_edge_attribute(index, ep, row[ep], type=attr_type) else: for index, row in df.iterrows(): #============= # ADD NODES #============= my_nicecx.create_node(id=row[0], node_name=row[0], node_represents=row[0]) my_nicecx.create_node(id=row[1], node_name=row[1], node_represents=row[1]) #============= # ADD EDGES #============= if len(row) > 2: my_nicecx.create_edge(id=index, edge_source=row[0], edge_target=row[1], edge_interaction=row[2]) else: my_nicecx.create_edge(id=index, edge_source=row[0], edge_target=row[1], edge_interaction='interacts-with') my_nicecx.add_metadata_stub('nodes') my_nicecx.add_metadata_stub('edges') if source_node_attr or target_node_attr: my_nicecx.add_metadata_stub('nodeAttributes') if edge_attr: my_nicecx.add_metadata_stub('edgeAttributes') return my_nicecx