def get_signor_update_mapping(server, username, password):
signor_mapping_set = {}
signor_mapping_list_df = get_signor_mapping_list_df()
signor_dict = signor_mapping_list_df.to_dict()
id_to_name = {}
pathway_names = signor_dict.get('pathway_name')
pathway_id = signor_dict.get('pathway_id')
for k, v in pathway_names.items():
signor_mapping_set[v] = k
id_to_name[pathway_id.get(k)] = v
my_ndex = nc.Ndex2(my_server, my_username, my_password)
networks = my_ndex.get_network_summaries_for_user(username)
update_mapping = {}
for nk in networks:
if nk.get('name') is not None:
if signor_mapping_set.get(nk.get('name')) is not None or 'FULL-' in nk.get('name').upper():
if 'FULL-HUMAN' in nk.get('name').upper():
update_mapping['FULL-Human (' + f"{datetime.now():%d-%b-%Y}" + ')'] = nk.get('externalId')
elif 'FULL-MOUSE' in nk.get('name').upper():
update_mapping['FULL-Mouse (' + f"{datetime.now():%d-%b-%Y}" + ')'] = nk.get('externalId')
elif 'FULL-RAT' in nk.get('name').upper():
update_mapping['FULL-Rat (' + f"{datetime.now():%d-%b-%Y}" + ')'] = nk.get('externalId')
else:
update_mapping[nk.get('name').upper()] = nk.get('externalId')
print(update_mapping)
return (update_mapping, id_to_name)
def process_full_signor(cytoscape_visual_properties_template_id, load_plan, server, username, password,
privateonly=False):
processed_uuids = []
species_mapping = {'9606': 'Human', '10090': 'Mouse', '10116': 'Rat'}
print('')
print('')
print('***********************************')
print('**** FULL NETWORKS ****')
print('***********************************')
for species_id in species:
print('')
print('Processing %s full network' % species_mapping.get(species_id))
network = get_full_signor_network(load_plan, species_id)
network.set_name('FULL-' + species_mapping.get(species_id) + ' (' + f"{datetime.now():%d-%b-%Y}" + ')')
if species_id == '9606':
network.set_network_attribute("organism", "Human, 9606, H**o sapiens")
elif species_id == '10090':
network.set_network_attribute("organism", "Mouse, 10090, Mus musculus")
elif species_id == '10116':
network.set_network_attribute("organism", "Rat, 10116, Rattus norvegicus")
network.apply_template(
username=username,
password=password,
server=server,
uuid=cytoscape_visual_properties_template_id)
#print('Applying spring layout')
#apply_spring_layout(network)
print('Skipping spring layout')
network_update_key = update_signor_mapping.get(network.get_name())
if network_update_key is not None:
print("updating")
#return \
upload_signor_network(network, server, username, password, update_uuid=network_update_key)
#processed_uuids.append(network_update_key)
else:
print("new network")
upload_message = upload_signor_network(network, server, username, password)
network_uuid = upload_message.split('/')[-1]
processed_uuids.append(network_uuid)
for sig_id in processed_uuids:
while True:
try:
my_ndex = nc.Ndex2(my_server, my_username, my_password)
if privateonly is False:
logger.info('Making network ' + sig_id +
' public, showcased, and searchable')
my_ndex._make_network_public_indexed(sig_id)
break
except Exception as excp:
print('Network not ready to be made PUBLIC. Sleeping...')
time.sleep(2)
return ''
def test_get_set(self):
username = '******'
password = '******'
server = 'dev.ndexbio.org'
my_network_set = '70800b06-29d2-11e7-8059-06832d634f41' # Test 5
ndex2_client = nc.Ndex2(host=server, username=username, password=password, debug=True)
set_response = ndex2_client.get_network_set(my_network_set)
self.assertTrue(set_response.get('externalId') is not None)
def get_update_mapping(self):
my_ndex = nc.Ndex2(self.params.get('server'), self.params.get('username'), self.params.get('password'))
networks = my_ndex.get_network_summaries_for_user(self.params.get('username'))
update_mapping = {}
for nk in networks:
if nk.get('name') is not None:
update_mapping[nk.get('name').upper()] = nk.get('externalId')
return update_mapping
def get_network_update_mapping(my_server, my_username, my_password):
my_ndex = nc2.Ndex2(my_server, my_username, my_password)
network_summaries = my_ndex.get_network_summaries_for_user(my_username)
update_mapping = {}
for nk in network_summaries:
if nk.get('name') is not None:
update_mapping[nk.get('name').upper()] = nk.get('externalId')
return update_mapping
def test_set_add(self):
username = '******'
password = '******'
server = 'dev.ndexbio.org'
my_network_set = '70800b06-29d2-11e7-8059-06832d634f41' # Test 5
my_test_networks = ['270e6bc1-d935-11e7-aa3d-06832d634f41']
ndex2_client = nc.Ndex2(host=server, username=username, password=password, debug=True)
set_response = ndex2_client.add_networks_to_networkset(my_network_set, my_test_networks)
self.assertTrue('http' in set_response) # Successful response is the URL to the set
def get_network_properties(params):
net_prop_ndex = nc2.Ndex2(params.get('server'), params.get('username'), params.get('password'))
network_properties_stream = net_prop_ndex.get_network_aspect_as_cx_stream(params.get('update_uuid'), 'networkAttributes')
network_properties = network_properties_stream.json()
return_properties = {}
for net_prop in network_properties:
return_properties[net_prop.get('n')] = net_prop.get('v')
return return_properties
def get_network_properties(server, username, password, network_id):
net_prop_ndex = nc2.Ndex2(server, username, password)
network_properties_stream = net_prop_ndex.get_network_aspect_as_cx_stream(network_id, 'networkAttributes')
network_properties = network_properties_stream.json()
return_properties = {}
for net_prop in network_properties:
return_properties[net_prop.get('n')] = net_prop.get('v')
return return_properties
def test_set_add_private(self):
username = '******'
password = '******'
server = 'dev.ndexbio.org'
my_network_set = '70800b06-29d2-11e7-8059-06832d634f41' # Test 5
my_test_networks = ['270e6bc1-d935-11e7-aa3d-06832d634f41']
ndex2_client = nc.Ndex2(host=server, username=username, password=password, debug=True)
set_response = ndex2_client.delete_networks_from_networkset(my_network_set, my_test_networks)
self.assertTrue(len(set_response) == 0) # Empty response means no errors
def __init__(self, server, username, password, **attr):
self.update_mapping = {}
self.server = server
self.username = username
self.password = password
self.network = None
my_ndex = nc.Ndex2(self.server, self.username, self.password)
networks = my_ndex.get_network_summaries_for_user(self.username)
for nk in networks:
if nk.get('name') is not None:
self.update_mapping[nk.get('name').upper()] = nk.get(
'externalId')
#!/usr/bin/env python
# coding: utf-8
# In[23]:
import ndex2.client as nc
import io
import json
from IPython.display import HTML
from time import sleep
anon_ndex = nc.Ndex2("http://public.ndexbio.org")
anon_ndex.update_status()
networks = anon_ndex.status.get("networkCount")
users = anon_ndex.status.get("userCount")
groups = anon_ndex.status.get("groupCount")
print("anon client: %s networks, %s users, %s groups" %
(networks, users, groups))
# In[18]:
from random import randrange
import ndex2.client as nc
import io
import json
from IPython.display import HTML
from time import sleep
ns = anon_ndex.get_network_summary('c9243cce-2d32-11e8-b939-0ac135e8bacf')
print('Processing ' + str(count + 1) + '/' + str(total_pathways))
upload_message = process_signor_id(pathway_id, cytoscape_visual_properties_template_id,
load_plan, my_server, my_username, my_password)
if upload_message is not None:
network_update_key = update_signor_mapping.get(signor_id_name_mapping.get(pathway_id).upper())
if network_update_key is None:
network_uuid = upload_message.split('/')[-1]
signor_uuids.append(network_uuid)
if limit:
count += 1
if args.privateonly is False:
my_ndex = nc.Ndex2(my_server, my_username, my_password)
for sig_id in signor_uuids:
logger.info('Making network ' + sig_id +
' public, showcased, and searchable')
my_ndex._make_network_public_indexed(sig_id)
logger.info('Done processing individual pathways.')
print('Done processing indiviual pathways.')
def process_full_signor(cytoscape_visual_properties_template_id, load_plan, server, username, password,
privateonly=False):
processed_uuids = []
species_mapping = {'9606': 'Human', '10090': 'Mouse', '10116': 'Rat'}
print('')
print('')
print('***********************************')
def process_rtx(rtx_json, network_name, username, password, server):
G = nx.Graph(name=network_name)
# ===========================
# CHECK FOR REQUIRED FIELDS
# ===========================
result_graph = rtx_json.get('result_graph')
if result_graph is not None:
n_data = result_graph.get('node_list')
e_data = result_graph.get('edge_list')
else:
raise Exception('ERROR: No result_graph found in input file')
# ==============================
# ADD NODES TO NETWORKX GRAPH
# ==============================
node_type_dict = {}
node_label_id_map = {}
edge_interaction_map = {}
layout_nlist = []
layout_level1_nlist = []
layout_level2_nlist = []
layout_other_nlist = []
for nds in n_data:
if nds.get('id') is not None and nds.get('type') is not None:
node_type_dict[nds.get('id')] = nds.get('type')
for n in n_data:
n_attrs = {}
node_label_id_map[n.get('id')] = n.get('name')
for k, v in n.items():
if k != 'id' and k != 'name':
n_attrs[k] = v
G.add_node(n.get('id'), n_attrs)
# Shell layout setup
if len(layout_level1_nlist) < 25:
layout_level1_nlist.append(n.get('id'))
elif len(layout_level2_nlist) < 50:
layout_level2_nlist.append(n.get('id'))
else:
layout_other_nlist.append(n.get('id'))
layout_nlist.append(layout_level1_nlist)
layout_nlist.append(layout_level2_nlist)
layout_nlist.append(layout_other_nlist)
# ==============================
# ADD EDGES TO NETWORKX GRAPH
# ==============================
edge_type_dict = {}
if e_data is not None:
for eds in e_data:
if eds.get('name') is not None and eds.get('type') is not None:
edge_type_dict[eds.get('name')] = eds.get('type')
for e in e_data:
#print(e)
e_attrs = {}
for k, v in e.items():
if k != 'source_id' and k != 'target_id' and k != 'id':
e_attrs[k] = v
if k == 'type':
if edge_interaction_map.get(e.get('source_id')) is None:
edge_interaction_map[e.get('source_id')] = {
e.get('target_id'): e.get('type')
}
else:
edge_interaction_map[e.get('source_id')][e.get(
'target_id')] = e.get('type')
if edge_interaction_map.get(e.get('target_id')) is None:
edge_interaction_map[e.get('target_id')] = {
e.get('source_id'): e.get('type')
}
else:
edge_interaction_map[e.get('target_id')][e.get(
'source_id')] = e.get('type')
G.add_edge(e.get('source_id'), e.get('target_id'), e_attrs)
else:
n1 = G.nodes()[0]
G.add_edge(n1, n1, {})
edge_interaction_map[n1] = {n1: 'self-ref'}
# ======================
# RUN NETWORKX LAYOUT
# ======================
init_pos = nx.drawing.circular_layout(G)
G.pos = nx.drawing.spring_layout(G,
iterations=20,
pos=init_pos,
weight='weight')
# G.pos = nx.drawing.shell_layout(G, nlist=layout_nlist)
# G.pos = nx.drawing.spring_layout(G, weight='weight')
# =========================
# CREATE CX FROM NETWORKX
# =========================
niceCx = ndex2.create_nice_cx_from_networkx(G)
niceCx.apply_template('public.ndexbio.org',
'84d64a82-23bc-11e8-b939-0ac135e8bacf')
# =========================
# SET NODE NAME TO LABEL
# =========================
for k, v in niceCx.nodes.items():
v.set_node_represents(v.get_name())
v.set_node_name(node_label_id_map.get(v.get_name()))
for k, v in niceCx.edges.items():
v.set_interaction(
edge_interaction_map.get(v.get_source()).get(v.get_target()))
# =======================
# UPLOAD TO NDEX SERVER
# =======================
if username is not None and password is not None and server is not None:
message = niceCx.upload_to(server, username, password)
network_uuid = message.split('/')[-1]
my_ndex = nc.Ndex2(server, username, password)
if 'error' not in message:
my_ndex._make_network_public_indexed(network_uuid)
return message
return None
# # NDEx2 Client Objects
# #### Modules Required for NDEx2 Client Tutorial
# In[ ]:
import ndex2.client as nc
import io
import json
from IPython.display import HTML
#create a NDEx client object to access my accouont on the NDEx server
my_account = "PUT YOUR USER HERE"
my_password = "******"
my_ndex = nc.Ndex2("http://public.ndexbio.org", my_account, my_password)
my_ndex.update_status()
# #### update_status()
# In[ ]:
try:
my_ndex = nc.Ndex2("http://public.ndexbio.org", my_account, my_password)
my_ndex.update_status()
networks = my_ndex.status.get("networkCount")
users = my_ndex.status.get("userCount")
groups = my_ndex.status.get("groupCount")
print("my_ndex client: %s networks, %s users, %s groups" %
(networks, users, groups))
except Exception as inst:
def _load_pcnet():
anon_ndex = nc.Ndex2("http://public.ndexbio.org")
network_niceCx = ndex2.create_nice_cx_from_server(server='public.ndexbio.org',
uuid='f93f402c-86d4-11e7-a10d-0ac135e8bacf')
return network_niceCx.to_networkx()
def process(node_list, edge_list, load_plan, network_name, username, password,
server):
try:
here = path.abspath(path.dirname(__file__))
with open(path.join(here, load_plan), 'r') as lp:
load_plan = json.load(lp)
with open(path.join(here, 'loading_plan_schema.json')) as schema_file:
plan_schema = json.load(schema_file)
validate(load_plan, plan_schema)
node_id = load_plan.get('source_plan').get('rep_column')
node_name = load_plan.get('source_plan').get('node_name_column')
edge_source = load_plan.get('edge_plan').get('edge_source')
edge_target = load_plan.get('edge_plan').get('edge_target')
predicate_id_column = load_plan.get('edge_plan').get(
'predicate_id_column')
default_edge_interaction = load_plan.get('edge_plan').get(
'default_predicate')
except jsonschema.ValidationError as e1:
print("Failed to parse the loading plan: " + e1.message)
print('at path: ' + str(e1.absolute_path))
print("in block: ")
print(e1.instance)
except Exception as ex1:
print("Failed to parse the loading plan: ")
G = nx.Graph(name=network_name)
# ==============================
# ADD NODES TO NETWORKX GRAPH
# ==============================
node_label_id_map = {}
for n in node_list:
n_attrs = {}
node_label_id_map[n.get(node_id)] = n.get(node_name)
for k, v in n.items():
if k != node_id and k != node_name:
n_attrs[k] = v
G.add_node(n.get(node_id), n_attrs)
# ==============================
# ADD EDGES TO NETWORKX GRAPH
# ==============================
for e in edge_list:
e_attrs = {}
interaction_found = False
for k, v in e.items():
if k != edge_source and k != edge_target and k != 'id':
e_attrs[k] = v
if predicate_id_column is not None and k == predicate_id_column:
interaction_found = True
set_edge_interaction_map(e.get(edge_source),
e.get(edge_target),
e.get(predicate_id_column))
#if edge_interaction_map.get(e.get(edge_source)) is None:
# edge_interaction_map[e.get(edge_source)] = {e.get(edge_target): e.get(predicate_id_column)}
#else:
# edge_interaction_map[e.get(edge_source)][e.get(edge_target)] = e.get(predicate_id_column)
#if edge_interaction_map.get(e.get(edge_target)) is None:
# edge_interaction_map[e.get(edge_target)] = {e.get(edge_source): e.get(predicate_id_column)}
#else:
# edge_interaction_map[e.get(edge_target)][e.get(edge_source)] = e.get(predicate_id_column)
if not interaction_found and default_edge_interaction is not None:
set_edge_interaction_map(e.get(edge_source), e.get(edge_target),
default_edge_interaction)
G.add_edge(e.get(edge_source), e.get(edge_target), e_attrs)
# ======================
# RUN NETWORKX LAYOUT
# ======================
G.pos = nx.drawing.spring_layout(G, iterations=20, weight='weight')
# =========================
# CREATE CX FROM NETWORKX
# =========================
niceCx = ndex2.create_nice_cx_from_networkx(G)
niceCx.apply_template('public.ndexbio.org',
'84d64a82-23bc-11e8-b939-0ac135e8bacf')
# =========================
# POST-PROCESS NODES
# SET NODE NAME TO LABEL
# =========================
for k, v in niceCx.nodes.items():
v.set_node_represents(v.get_name())
v.set_node_name(node_label_id_map.get(v.get_name()))
for k, v in niceCx.edges.items():
if predicate_id_column is None:
v.set_interaction(default_edge_interaction)
else:
v.set_interaction(
edge_interaction_map.get(v.get_source()).get(v.get_target()))
# =======================
# UPLOAD TO NDEX SERVER
# =======================
if username is not None and password is not None and server is not None:
message = niceCx.upload_to(server, username, password)
if 'error' not in message:
network_uuid = message.split('/')[-1]
my_ndex = nc.Ndex2(server, username, password)
my_ndex._make_network_public_indexed(network_uuid)
return message
return None
def __init__(self):
logger.debug('Starting QCA')
self.host = "http://www.ndexbio.org"
self.results_directory = "qca_results"
self.directed_path_query_url = \
'http://general.bigmech.ndexbio.org:5603/directedpath/query'
self.context_expression_query_url = \
'http://general.bigmech.ndexbio.org:8081' + \
'/context/expression/cell_line'
self.context_mutation_query_url = \
'http://general.bigmech.ndexbio.org:8081/' + \
'context/mutation/cell_line'
# dict of reference network descriptors by network name
self.reference_networks = [
#{
# "id": "84f321c6-dade-11e6-86b1-0ac135e8bacf",
# "name": "prior",
# "type": "canonical",
# "server": "public.ndexbio.org"
#}
{
#"id": "d68677b8-173d-11e7-b39e-0ac135e8bacf",
"id": "50e3dff7-133e-11e6-a039-06603eb7f303",
#"id": "89274295-1730-11e7-b39e-0ac135e8bacf",
"name": "Ras Machine",
"type": "canonical",
"server": "public.ndexbio.org"
}
]
# --------------------------
# Schemas
self.query_result_schema = {
"network_description": {},
"forward_paths": [],
"reverse_paths": [],
"forward_mutation_paths": [],
"reverse_mutation_paths": []
}
self.reference_network_schema = {
"id": "",
"name": "",
"type": "canonical"
}
self.query_schema = {
"source_names": [],
"target_names": [],
"cell_line": ""
}
# --------------------------
# Cell Lines
self.cell_lines = []
# --------------------------
# Queries
# list of query dicts
self.queries = []
try:
self.ndex = nc.Ndex2(host=self.host)
except Exception as e:
logger.error('QCA could not connect to %s' % self.host)
logger.error(e)
self.ndex = None
terms_small = [t for t,s in zip(ont.terms, ont.term_sizes) if (s >= args.subnet_size[0]) and (s<args.evinet_size)]
terms_big = [t for t,s in zip(ont.terms, ont.term_sizes) if (s >= args.evinet_size) and (s<=args.subnet_size[1])]
# upload/reuse subnetworks
all_uuid = []
term_uuid = {}
if args.score != None:
term_uuid.update(create_term_to_uuid(ont, args.hier_name, terms_big, None, args.subnet_size, args.score, uuid_file=args.term_2_uuid))
term_uuid.update(create_term_to_uuid(ont, args.hier_name, terms_small, args.evinet_links, args.subnet_size, args.score, uuid_file=args.term_2_uuid))
for u in term_uuid.values():
all_uuid.append(u)
# upload the main hierarchy
if not args.skip_main:
# now if visible cols not specified, visualize all the columns in node attribute
vis_cols = []
if args.visible_cols == None:
vis_cols = node_attr.columns.tolist()
else:
vis_cols = args.visible_cols
main_uuid = upload_main_hierarchy(ont, args.hier_name, term_uuid, vis_cols)
all_uuid.append(main_uuid)
# create a network set containing everything uploaded for this hierarchy
my_ndex = nc2.Ndex2(args.ndex_account[0], args.ndex_account[1], args.ndex_account[2])
my_ndex.update_status()
url = my_ndex.create_networkset(args.hier_name, '')
set_uuid = url.split('/')[-1]
my_ndex.add_networks_to_networkset(set_uuid, all_uuid) # this function is only available in Ndex2
# TODO: cannot have duplicated set name
def __init__(self, path_host=None, network_uuid=None):
logger.debug('Starting QCA')
if not path_host:
path_host = '34.230.33.149'
if not network_uuid:
network_uuid = '50e3dff7-133e-11e6-a039-06603eb7f303'
logger.info('Using host %s and network %s' % (path_host, network_uuid))
self.host = "http://www.ndexbio.org"
self.results_directory = "qca_results"
self.directed_path_query_url = \
('http://%s:5603/directedpath/query' % path_host)
self.context_expression_query_url = \
'http://general.bigmech.ndexbio.org:8081' + \
'/context/expression/cell_line'
self.context_mutation_query_url = \
'http://general.bigmech.ndexbio.org:8081/' + \
'context/mutation/cell_line'
# dict of reference network descriptors by network name
self.reference_networks = [
#{
# "id": "84f321c6-dade-11e6-86b1-0ac135e8bacf",
# "name": "prior",
# "type": "canonical",
# "server": "public.ndexbio.org"
#}
{
# Large network from preassembled DB
# "id": "04020c47-4cfd-11e8-a4bf-0ac135e8bacf",
# The RAS Machine network
#"id": "50e3dff7-133e-11e6-a039-06603eb7f303",
#"id": "d68677b8-173d-11e7-b39e-0ac135e8bacf",
#"id": "89274295-1730-11e7-b39e-0ac135e8bacf",
'id': network_uuid,
"name": "Ras Machine",
"type": "canonical",
"server": "public.ndexbio.org"
}
]
# --------------------------
# Schemas
self.query_result_schema = {
"network_description": {},
"forward_paths": [],
"reverse_paths": [],
"forward_mutation_paths": [],
"reverse_mutation_paths": []
}
self.reference_network_schema = {
"id": "",
"name": "",
"type": "canonical"
}
self.query_schema = {
"source_names": [],
"target_names": [],
"cell_line": ""
}
# --------------------------
# Cell Lines
self.cell_lines = []
# --------------------------
# Queries
# list of query dicts
self.queries = []
try:
self.ndex = nc.Ndex2(host=self.host)
except Exception as e:
logger.error('QCA could not connect to %s' % self.host)
logger.error(e)
self.ndex = None
current_directory = path.dirname(path.abspath(__file__))
biopax_path = path.join(current_directory, 'biopax')
sif_path = path.join(current_directory, 'biopax', 'sif')
biopax_files = listdir(biopax_path)
grp_file_path = path.join(current_directory, 'grp')
paxtools_file = path.join(current_directory, 'paxtools.jar')
#======================================
#======================================
# Process biopax files into ebs files
#======================================
#======================================
if True:
my_ndex = nc.Ndex2('dev.ndexbio.org', 'scratch', 'scratch')
network_properties_stream = my_ndex.get_network_aspect_as_cx_stream(
'c142dc09-28a1-11e8-9c1a-525400c25d22', 'networkAttributes')
network_properties = network_properties_stream.json()
return_properties = {}
for net_prop in network_properties:
return_properties[net_prop.get('n')] = net_prop.get('v')
print(network_properties)
if False:
for bpf in biopax_files:
if path.isfile(path.join(
biopax_path, bpf)) and bpf != '.DS_Store' and 'zip' not in bpf:
