def create_from_server(self, server, username, password, uuid): if server and uuid: niceCx = NiceCXNetwork() #=================== # METADATA #=================== available_aspects = [] for ae in (o for o in self.stream_aspect(uuid, 'metaData')): available_aspects.append(ae.get(CX_CONSTANTS.METADATA_NAME)) mde = MetaDataElement(json_obj=ae) niceCx.add_metadata(mde) #available_aspects = ['edges', 'nodes'] # TODO - remove this opaque_aspects = set(available_aspects).difference(known_aspects_min) print(opaque_aspects) #==================== # NETWORK ATTRIBUTES #==================== objects = self.stream_aspect(uuid, 'networkAttributes') obj_items = (o for o in objects) for network_item in obj_items: add_this_network_attribute = NetworkAttributesElement(cx_fragment=network_item) niceCx.add_network_attribute(add_this_network_attribute) #=================== # NODES #=================== objects = self.stream_aspect(uuid, 'nodes') obj_items = (o for o in objects) for node_item in obj_items: add_this_node = NodeElement(cx_fragment=node_item) niceCx.create_node(add_this_node) #=================== # EDGES #=================== objects = self.stream_aspect(uuid, 'edges') obj_items = (o for o in objects) for edge_item in obj_items: add_this_edge = EdgeElement(cx_fragment=edge_item) niceCx.create_edge(add_this_edge) #=================== # NODE ATTRIBUTES #=================== objects = self.stream_aspect(uuid, 'nodeAttributes') obj_items = (o for o in objects) for att in obj_items: add_this_node_att = NodeAttributesElement(json_obj=att) niceCx.add_node_attribute(add_this_node_att) #=================== # EDGE ATTRIBUTES #=================== objects = self.stream_aspect(uuid, 'edgeAttributes') obj_items = (o for o in objects) for att in obj_items: add_this_edge_att = EdgeAttributesElement(json_obj=att) niceCx.add_edge_attribute(add_this_edge_att) #=================== # CITATIONS #=================== objects = self.stream_aspect(uuid, 'citations') obj_items = (o for o in objects) for cit in obj_items: add_this_citation = CitationElement(cx_fragment=cit) niceCx.add_citation(add_this_citation) #=================== # SUPPORTS #=================== objects = self.stream_aspect(uuid, 'supports') obj_items = (o for o in objects) for sup in obj_items: add_this_supports = SupportElement(cx_fragment=sup) niceCx.add_support(add_this_supports) #=================== # NODE CITATIONS #=================== objects = self.stream_aspect(uuid, 'nodeCitations') obj_items = (o for o in objects) for node_cit in obj_items: niceCx.add_node_citations_from_cx(node_cit) #=================== # EDGE CITATIONS #=================== objects = self.stream_aspect(uuid, 'edgeCitations') obj_items = (o for o in objects) for edge_cit in obj_items: niceCx.add_edge_citations_from_cx(edge_cit) #=================== # OPAQUE ASPECTS #=================== for oa in opaque_aspects: objects = self.stream_aspect(uuid, oa) obj_items = (o for o in objects) for oa_item in obj_items: aspect_element = AspectElement(oa_item, oa) niceCx.add_opaque_aspect_element(aspect_element) return niceCx else: raise Exception('Server and uuid not specified')
def query_network(self, uuid, search_string, max_edges): myConst = CX_CONSTANTS niceCx = NiceCXNetwork() #uuid = '7246d8cf-c644-11e6-b48c-0660b7976219' search_terms_dict = {k:1 for k in search_string.split(',')} solr = pysolr.Solr(solr_url + uuid + '/', timeout=10) try: results = solr.search(search_string, rows=10000) #search_terms_array = [int(n['id']) for n in results.docs] search_terms_array = {int(n['id']):1 for n in results.docs} if(not search_terms_array): return {'message': 'No nodes found'} print('starting nodes 1') #=================== # METADATA #=================== available_aspects = [] for ae in (o for o in self.stream_aspect(uuid, 'metaData')): available_aspects.append(ae.get(CX_CONSTANTS.METADATA_NAME)) mde = MetaDataElement(json_obj=ae) niceCx.add_metadata(mde) #available_aspects = ['edges', 'nodes'] # TODO - remove this opaque_aspects = set(available_aspects).difference(known_aspects_min) print(opaque_aspects) #=================== # NODES #=================== if 'nodes' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'nodes')): if search_terms_array.get(ae.get(CX_CONSTANTS.ID)): add_this_node = NodeElement(cx_fragment=ae) niceCx.create_node(add_this_node) else: raise Exception('Network does not contain any nodes. Cannot query') print('starting edges 1') #=================== # EDGES #=================== edge_count = 0 added_edges = 0 start_time = time.time() if 'edges' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'edges')): if niceCx.nodes.get(ae.get(CX_CONSTANTS.EDGE_SOURCE_NODE_ID_OR_SUBNETWORK)) is not None or niceCx.nodes.get(ae.get(CX_CONSTANTS.EDGE_TARGET_NODE_ID)) is not None: add_this_edge = EdgeElement(cx_fragment=ae) niceCx.create_edge(add_this_edge) added_edges += 1 if edge_count % 5000 == 0: print(edge_count) #if edge_count > 30000: # break if added_edges > max_edges: raise StopIteration('Max edges reached') edge_count += 1 else: raise Exception('Network does not contain any nodes. Cannot query') print('Response time (Edge search): ' + str(time.time() - start_time)) print('starting nodes 2') #=================== # NODES #=================== for ae in (o for o in self.stream_aspect(uuid, 'nodes')): if niceCx.get_missing_nodes().get(ae.get(CX_CONSTANTS.ID)): add_this_node = NodeElement(cx_fragment=ae) niceCx.create_node(add_this_node) #==================== # NETWORK ATTRIBUTES #==================== if 'networkAttributes' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'networkAttributes')): add_this_network_attribute = NetworkAttributesElement(cx_fragment=ae) niceCx.add_network_attribute(add_this_network_attribute) #=================== # NODE ATTRIBUTES #=================== if 'nodeAttributes' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'nodeAttributes')): if niceCx.nodes.get(ae.get(CX_CONSTANTS.PROPERTY_OF)): add_this_node_att = NodeAttributesElement(json_obj=ae) niceCx.add_node_attribute(add_this_node_att) #=================== # EDGE ATTRIBUTES #=================== if 'edgeAttributes' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'edgeAttributes')): if niceCx.edges.get(ae.get(CX_CONSTANTS.PROPERTY_OF)): add_this_edge_att = EdgeAttributesElement(json_obj=ae) niceCx.set_edge_attribute() #=================== # NODE CITATIONS #=================== if 'nodeCitations' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'nodeCitations')): for e_po in ae.get(CX_CONSTANTS.PROPERTY_OF): if niceCx.get_nodes().get(e_po) is not None: niceCx.add_node_citations_from_cx(ae) #=================== # EDGE CITATIONS #=================== ec_count = 0 if 'edgeCitations' in available_aspects: for ae in (o for o in self.stream_aspect(uuid, 'edgeCitations')): for e_po in ae.get(CX_CONSTANTS.PROPERTY_OF): if niceCx.get_edges().get(e_po) is not None: niceCx.add_edge_citations_from_cx(ae) ec_count += 1 if ec_count % 500 == 0: print(ec_count) #=================== # CITATIONS #=================== if 'citations' in available_aspects: #====================================================== # FILTER CITATIONS IF THERE ARE EDGE OR NODE CITATIONS # OTHERWISE ADD THEM ALL (NO-FILTER) -- TODO #====================================================== for ae in (o for o in self.stream_aspect(uuid, 'citations')): add_this_citation = CitationElement(cx_fragment=ae) niceCx.add_citation(add_this_citation) #=================== # OPAQUE ASPECTS #=================== for oa in opaque_aspects: objects = self.stream_aspect(uuid, oa) obj_items = (o for o in objects) for oa_item in obj_items: aspect_element = AspectElement(oa_item, oa) niceCx.add_opaque_aspect(aspect_element) except SolrError as se: if('404' in se.message): ndex2.get_logger('SOLR').warning('Network not found ' + self.uuid + ' on ' + solr_url + ' server.') raise Exception("Network not found (SOLR)") else: ndex2.get_logger('SOLR').warning('Network error ' + self.uuid + ' on ' + solr_url + ' server. ' + se.message) raise Exception(se.message) except StopIteration as si: ndex2.get_logger('QUERY').warning("Found more than max edges. Raising exception") raise StopIteration(si.message) #nice_cx_json = niceCx.to_cx() return niceCx
def test_nice_cx_model(self): niceCx = NiceCXNetwork() #main_map = NdexGraph(server='http://dev2.ndexbio.org', username='******', password='******', uuid='7246d8cf-c644-11e6-b48c-0660b7976219') uuid = '6b968fd2-02a4-11e6-b550-06603eb7f303' #==================== # NETWORK QUERY #==================== networkQuery = NetworkQuery() networkQuery.query_network('40f1def0-3aa4-11e7-b12f-0660b7976219', 'HSPA5,HSPA4') my_na = NodeAttributesElement( subnetwork=1, property_of=11, name=22, values=33, type=ATTRIBUTE_DATA_TYPE.convert_to_data_type('string')) #==================== # NETWORK ATTRIBUTES #==================== #objects = ijson.items(urlopen('http://dev2.ndexbio.org/v2/network/' + uuid + '/aspect/nodes'), 'item') objects = loadAspect('networkAttributes') obj_items = (o for o in objects) for network_item in obj_items: add_this_network_attribute = NetworkAttributesElement( cx_fragment=network_item) niceCx.add_network_attribute(add_this_network_attribute) #=================== # NODES #=================== #objects = ijson.items(urlopen('http://dev2.ndexbio.org/v2/network/' + uuid + '/aspect/nodes'), 'item') objects = loadAspect('nodes') obj_items = (o for o in objects) for node_item in obj_items: add_this_node = NodeElement(json_obj=node_item) niceCx.create_node(add_this_node) #=================== # EDGES #=================== #objects = ijson.items(urlopen('http://dev2.ndexbio.org/v2/network/' + uuid + '/aspect/edges'), 'item') objects = loadAspect('edges') obj_items = (o for o in objects) for edge_item in obj_items: add_this_edge = EdgeElement(json_obj=edge_item) niceCx.create_edge(add_this_edge) #=================== # NODE ATTRIBUTES #=================== #objects = ijson.items(urlopen('http://dev2.ndexbio.org/v2/network/' + uuid + '/aspect/nodeAttributes'), 'item') objects = loadAspect('nodeAttributes') obj_items = (o for o in objects) for att in obj_items: add_this_node_att = NodeAttributesElement(json_obj=att) niceCx.add_node_attribute(add_this_node_att) #=================== # EDGE ATTRIBUTES #=================== #objects = ijson.items(urlopen('http://dev2.ndexbio.org/v2/network/' + uuid + '/aspect/edgeAttributes'), 'item') objects = loadAspect('edgeAttributes') obj_items = (o for o in objects) for att in obj_items: add_this_edge_att = EdgeAttributesElement(json_obj=att) niceCx.add_edge_attribute(add_this_edge_att) #=================== # CITATIONS #=================== #objects = ijson.items(urlopen('http://dev2.ndexbio.org/v2/network/' + uuid + '/aspect/edgeAttributes'), 'item') objects = loadAspect('citations') obj_items = (o for o in objects) for cit in obj_items: add_this_citation = CitationElement(cx_fragment=cit) niceCx.add_citation(add_this_citation) #=================== # SUPPORTS #=================== objects = loadAspect('supports') obj_items = (o for o in objects) for sup in obj_items: add_this_supports = SupportElement(cx_fragment=sup) niceCx.add_support(add_this_supports) #=================== # NODE CITATIONS #=================== objects = loadAspect('nodeCitations') obj_items = (o for o in objects) for node_cit in obj_items: niceCx.add_node_citations_from_cx(node_cit) #=================== # EDGE CITATIONS #=================== objects = loadAspect('edgeCitations') obj_items = (o for o in objects) for edge_cit in obj_items: niceCx.add_edge_citations_from_cx(edge_cit) nice_cx_json = niceCx.to_cx() # serialized = pickle.dumps(niceCx, protocol=0) # print 'Serialized memory:', sys.getsizeof(serialized) print('starting to_pandas_dataframe') niceCx.to_pandas_dataframe() parser = ijson.parse( urlopen('http://dev2.ndexbio.org/v2/network/' + uuid + '/aspect/nodes')) node_id, node_n, node_r = '', '', '' edge_id, edge_s, edge_t, edge_i = '', '', '', '' node_matches = {} edge_matches = {} edge_connected = {} node_found = False edge_found = False count = 0 done_searching = False start_time = time.time() for prefix, event, value in parser: if (prefix) == ('item.@id'): if count % 10000 == 0: print(count) count += 1 node_id = value elif (prefix) == ('item.n'): node_n = value node_found = True elif (prefix) == ('item.r'): node_r = value if node_found: node_matches[node_id] = {'n': node_n, 'r': node_r} add_this_node = NodeElement(id=node_id, node_name=node_n, node_represents=node_r) niceCx.create_node(add_this_node) node_found = False else: # No represents found if node_found: node_matches[node_id] = {'n': node_n} add_this_node = NodeElement(id=node_id, node_name=node_n) niceCx.create_node(add_this_node) node_found = False print('Response time (Node search): ' + str(time.time() - start_time)) start_time = time.time() print(edge_matches) print(node_matches) parser = ijson.parse( urlopen('http://dev2.ndexbio.org/v2/network/' + uuid + '/aspect/edges')) for prefix, event, value in parser: if (prefix) == ('item.@id'): edge_id = value elif (prefix) == ('item.s'): edge_s = value edge_found = True elif (prefix) == ('item.t'): edge_t = value edge_found = True elif (prefix) == ('item.i'): edge_i = value if edge_found: edge_matches[edge_id] = { 's': edge_s, 't': edge_t, 'i': edge_i } add_this_edge = EdgeElement(id=edge_id, edge_source=edge_s, edge_target=edge_t, edge_interaction=edge_i) niceCx.create_edge(add_this_edge) edge_connected[edge_s] = 1 edge_connected[edge_t] = 1 edge_found = False else: # No interaction found if edge_found: edge_matches[edge_id] = {'s': edge_s, 't': edge_t} add_this_edge = EdgeElement(id=edge_id, edge_source=edge_s, edge_target=edge_t) niceCx.create_edge(add_this_edge) edge_connected[edge_s] = 1 edge_connected[edge_t] = 1 edge_found = False print('Response time (Edge search): ' + str(time.time() - start_time)) start_time = time.time() print(edge_matches) print(node_matches) self.assertTrue(niceCx is not None)