def _setup_signed_graph():
edges, signed_edges, edge_beliefs, all_ns = _digraph_setup()
seg = nx.MultiDiGraph()
seg.add_edges_from(signed_edges)
# ATTN!! seg.edges yields u, v, index while seg.edges() yields u, v
for u, v, sign in seg.edges:
seg.edges[(u, v, sign)]['sign'] = sign
seg.edges[(u, v, sign)]['belief'] = edge_beliefs[(u, v)]
for node, data in seg.nodes(data=True):
data['ns'] = node[0]
data['id'] = node[1]
sng = signed_edges_to_signed_nodes(graph=seg,
prune_nodes=True,
copy_edge_data=True)
for u, v in sng.edges:
sng.edges[(u, v)]['weight'] = -np.log(sng.edges[(u, v)]['belief'])
return seg, sng, all_ns
def db_dump_to_pybel_sg(stmts_list=None,
pybel_model=None,
belief_dump=None,
default_belief=0.1,
sign_in_edges=False):
"""Create a signed pybel graph from an evidenceless dump from the db
Parameters
----------
stmts_list : list[indra.statements.Statement]
Provide a list of statements if they are already loaded. By default
the latest available pa statements dump is downloaded from s3.
Default: None.
pybel_model : pybel.BELGraph
If provided, skip generating a new pybel model from scratch
belief_dump : dict
If provided, reset the belief scores associated with the statements
supporting the edges.
default_belief : float
Only used if belief_dump is provided. When no belief score is
available, reset to this belief score. Default: 0.1.
sign_in_edges : bool
If True, check that all edges are stored with an index corresponding
to the sign of the edge. Default: False.
Returns
-------
tuple(DiGraph, MultiDiGraph)
"""
# Get statement dump:
# Look for latest file on S3 and pickle.loads it
if pybel_model is None:
pb_model = _custom_pb_assembly(stmts_list)
else:
logger.info('Pybel model provided')
pb_model = pybel_model
# If belief dump is provided, reset beliefs to the entries in it
if belief_dump:
logger.info('Belief dump provided, resetting belief scores')
missing_hash = 0
changed_belief = 0
no_hash = 0
logger.info(f'Looking for belief scores among {len(pb_model.edges)} '
f'edges')
for edge in pb_model.edges:
ed = pb_model.edges[edge]
if ed and ed.get('stmt_hash'):
h = ed['stmt_hash']
if h in belief_dump:
ed['belief'] = belief_dump[h]
changed_belief += 1
else:
logger.warning(f'No belief found for {h}')
ed['belief'] = default_belief
missing_hash += 1
else:
no_hash += 1
logger.info(f'{no_hash} edges did not have hashes')
logger.info(f'{changed_belief} belief scores were changed')
logger.info(f'{missing_hash} edges did not have a belief entry')
# Get a signed edge graph
logger.info('Getting a PyBEL signed edge graph')
pb_signed_edge_graph = belgraph_to_signed_graph(
pb_model,
symmetric_variant_links=True,
symmetric_component_links=True,
propagate_annotations=True)
if sign_in_edges:
for u, v, ix in pb_signed_edge_graph.edges:
ed = pb_signed_edge_graph.edges[(u, v, ix)]
if 'sign' in ed and ix != ed['sign']:
pb_signed_edge_graph.add_edge(u, v, ed['sign'], **ed)
pb_signed_edge_graph.remove_edge(u, v, ix)
# Map hashes to edges
logger.info('Getting hash to signed edge mapping')
seg_hash_edge_dict = {}
for edge in pb_signed_edge_graph.edges:
if pb_signed_edge_graph.edges[edge].get('stmt_hash'):
seg_hash_edge_dict[pb_signed_edge_graph.edges[edge]
['stmt_hash']] = edge
pb_signed_edge_graph.graph['edge_by_hash'] = seg_hash_edge_dict
# Get the signed node graph
logger.info('Getting a signed node graph from signed edge graph')
pb_signed_node_graph = signed_edges_to_signed_nodes(pb_signed_edge_graph,
copy_edge_data=True)
# Map hashes to edges for signed nodes
logger.info('Getting hash to edge mapping')
sng_hash_edge_dict = {}
for edge in pb_signed_node_graph.edges:
if pb_signed_node_graph.edges[edge].get('stmt_hash'):
sng_hash_edge_dict[pb_signed_node_graph.edges[edge]
['stmt_hash']] = edge
pb_signed_node_graph.graph['edge_by_hash'] = sng_hash_edge_dict
logger.info('Done assembling signed edge and signed node PyBEL graphs')
return pb_signed_edge_graph, pb_signed_node_graph
def sif_dump_df_to_digraph(df: Union[pd.DataFrame, str],
date: str,
mesh_id_dict: Optional[Dict] = None,
graph_type: GraphTypes = 'digraph',
include_entity_hierarchies: bool = True,
sign_dict: Optional[Dict[str, int]] = None,
stmt_types: Optional[List[str]] = None,
z_sc_path: Optional[Union[str, pd.DataFrame]] = None,
verbosity: int = 0) \
-> Union[DiGraph, MultiDiGraph, Tuple[MultiDiGraph, DiGraph]]:
"""Return a NetworkX digraph from a pandas dataframe of a db dump
Parameters
----------
df : Union[str, pd.DataFrame]
A dataframe, either as a file path to a file (.pkl or .csv) or a
pandas DataFrame object.
date : str
A date string specifying when the data was dumped from the database.
mesh_id_dict : dict
A dict object mapping statement hashes to all mesh ids sharing a
common PMID
graph_type : str
Return type for the returned graph. Currently supports:
- 'digraph': DiGraph (Default)
- 'multidigraph': MultiDiGraph
- 'signed': Tuple[DiGraph, MultiDiGraph]
- 'signed-expanded': Tuple[DiGraph, MultiDiGraph]
- 'digraph-signed-types': DiGraph
include_entity_hierarchies : bool
If True, add edges between nodes if they are related ontologically
with stmt type 'fplx': e.g. BRCA1 is in the BRCA family, so an edge
is added between the nodes BRCA and BRCA1. Default: True. Note that
this option only is available for the options directed/unsigned graph
and multidigraph.
sign_dict : Dict[str, int]
A dictionary mapping a Statement type to a sign to be used for the
edge. By default only Activation and IncreaseAmount are added as
positive edges and Inhibition and DecreaseAmount are added as
negative edges, but a user can pass any other Statement types in a
dictionary.
stmt_types : List[str]
A list of statement types to epxand out to other signs
z_sc_path:
If provided, must be or be path to a square dataframe with HGNC symbols
as names on the axes and floats as entries
verbosity: int
Output various messages if > 0. For all messages, set to 4.
Returns
-------
Union[DiGraph, MultiDiGraph, Tuple[DiGraph, MultiDiGraph]]
The type is determined by the graph_type argument
"""
graph_options = ('digraph', 'multidigraph', 'signed', 'signed-expanded',
'digraph-signed-types')
if graph_type.lower() not in graph_options:
raise ValueError(f'Graph type {graph_type} not supported. Can only '
f'chose between {graph_options}')
sign_dict = sign_dict if sign_dict else default_sign_dict
graph_type = graph_type.lower()
date = date if date else datetime.now().strftime('%Y-%m-%d')
if isinstance(df, str):
sif_df = file_opener(df)
else:
sif_df = df
if z_sc_path is not None:
if isinstance(z_sc_path, str):
if z_sc_path.endswith('h5'):
logger.info(f'Loading z-scores from {z_sc_path}')
z_sc_df = pd.read_hdf(z_sc_path)
elif z_sc_path.endswith('pkl'):
logger.info(f'Loading z-scores from {z_sc_path}')
z_sc_df: pd.DataFrame = file_opener(z_sc_path)
else:
raise ValueError(f'Unrecognized file: {z_sc_path}')
elif isinstance(z_sc_path, pd.DataFrame):
z_sc_df = z_sc_path
else:
raise ValueError('Only file paths and data frames allowed as '
'arguments to z_sc_path')
else:
z_sc_df = None
# If signed types: filter out rows that of unsigned types
if graph_type == 'digraph-signed-types':
sif_df = sif_df[sif_df.stmt_type.isin(sign_dict.keys())]
sif_df = sif_dump_df_merger(sif_df,
graph_type,
sign_dict,
stmt_types,
mesh_id_dict,
verbosity=verbosity)
# Map ns:id to node name
logger.info('Creating dictionary mapping (ns,id) to node name')
ns_id_name_tups = set(zip(
sif_df.agA_ns, sif_df.agA_id, sif_df.agA_name)).union(
set(zip(sif_df.agB_ns, sif_df.agB_id, sif_df.agB_name)))
ns_id_to_nodename = {(ns, _id): name for ns, _id, name in ns_id_name_tups}
# Map hashes to edge for non-signed graphs
if graph_type in {'multidigraph', 'digraph', 'digraph-signed-types'}:
logger.info('Creating dictionary mapping hashes to edges for '
'unsigned graph')
hash_edge_dict = {
h: (a, b)
for a, b, h in zip(sif_df.agA_name, sif_df.agB_name,
sif_df.stmt_hash)
}
# Create graph from df
if graph_type == 'multidigraph':
indranet_graph = IndraNet.from_df(sif_df)
elif graph_type in ('digraph', 'digraph-signed-types'):
# Flatten
indranet_graph = IndraNet.digraph_from_df(sif_df,
'complementary_belief',
_weight_mapping)
elif graph_type in ('signed', 'signed-expanded'):
signed_edge_graph: MultiDiGraph = IndraNet.signed_from_df(
df=sif_df,
flattening_method='complementary_belief',
weight_mapping=_weight_mapping)
signed_node_graph: DiGraph = signed_edges_to_signed_nodes(
graph=signed_edge_graph, copy_edge_data=True)
signed_edge_graph.graph['date'] = date
signed_node_graph.graph['date'] = date
signed_edge_graph.graph['node_by_ns_id'] = ns_id_to_nodename
signed_node_graph.graph['node_by_ns_id'] = ns_id_to_nodename
# Get hash to signed edge mapping
logger.info('Creating dictionary mapping hashes to edges for '
'unsigned graph')
seg_hash_edge_dict = {} if graph_type == 'signed' else defaultdict(set)
for edge in signed_edge_graph.edges:
for es in signed_edge_graph.edges[edge]['statements']:
if graph_type == 'signed':
seg_hash_edge_dict[es['stmt_hash']] = edge
else:
seg_hash_edge_dict[es['stmt_hash']].add(edge)
signed_edge_graph.graph['edge_by_hash'] = seg_hash_edge_dict
sng_hash_edge_dict = {} if graph_type == 'signed' else defaultdict(set)
for edge in signed_node_graph.edges:
for es in signed_node_graph.edges[edge]['statements']:
if graph_type == 'signed':
sng_hash_edge_dict[es['stmt_hash']] = edge
else:
sng_hash_edge_dict[es['stmt_hash']].add(edge)
signed_node_graph.graph['edge_by_hash'] = sng_hash_edge_dict
if z_sc_df is not None:
# Set z-score attributes
add_corr_to_edges(graph=signed_edge_graph, z_corr=z_sc_df)
add_corr_to_edges(graph=signed_node_graph, z_corr=z_sc_df)
return signed_edge_graph, signed_node_graph
else:
raise ValueError(f'Unrecognized graph type {graph_type}. Must be one '
f'of: {", ".join(graph_options)}')
if z_sc_df is not None:
# Set z-score attributes
add_corr_to_edges(graph=indranet_graph, z_corr=z_sc_df)
# Add hierarchy relations to graph (not applicable for signed graphs)
if include_entity_hierarchies and graph_type in ('multidigraph',
'digraph'):
from depmap_analysis.network_functions.famplex_functions import \
get_all_entities
logger.info('Fetching entity hierarchy relationships')
full_entity_list = get_all_entities()
logger.info('Adding entity hierarchy manager as graph attribute')
node_by_uri = {uri: _id for (ns, _id, uri) in full_entity_list}
added_pairs = set() # Save (A, B, URI)
logger.info('Building entity relations to be added to data frame')
entities = 0
non_corr_weight = None
if z_sc_df is not None:
# Get non-corr weight
for edge in indranet_graph.edges:
if indranet_graph.edges[edge]['z_score'] == 0:
non_corr_weight = indranet_graph.edges[edge]['corr_weight']
break
assert non_corr_weight is not None
z_sc_attrs = {'z_score': 0, 'corr_weight': non_corr_weight}
else:
z_sc_attrs = {}
for ns, _id, uri in full_entity_list:
node = _id
# Get name in case it's different than id
if ns_id_to_nodename.get((ns, _id), None):
node = ns_id_to_nodename[(ns, _id)]
else:
ns_id_to_nodename[(ns, _id)] = node
# Add famplex edge
for pns, pid in bio_ontology.get_parents(ns, _id):
puri = get_identifiers_url(pns, pid)
pnode = pid
if ns_id_to_nodename.get((pns, pid), None):
pnode = ns_id_to_nodename[(pns, pid)]
else:
ns_id_to_nodename[(pns, pid)] = pnode
# Check if edge already exists
if (node, pnode, puri) not in added_pairs:
entities += 1
# Belief and evidence are conditional
added_pairs.add((node, pnode, puri)) # A, B, uri of B
ed = {
'agA_name': node,
'agA_ns': ns,
'agA_id': _id,
'agB_name': pnode,
'agB_ns': pns,
'agB_id': pid,
'stmt_type': 'fplx',
'evidence_count': 1,
'source_counts': {
'fplx': 1
},
'stmt_hash': puri,
'belief': 1.0,
'weight': MIN_WEIGHT,
'curated': True,
'english': f'{pns}:{pid} is an ontological parent '
f'of {ns}:{_id}',
'z_score': 0,
'corr_weight': 1
}
# Add non-existing nodes
if ed['agA_name'] not in indranet_graph.nodes:
indranet_graph.add_node(ed['agA_name'],
ns=ed['agA_ns'],
id=ed['agA_id'])
if ed['agB_name'] not in indranet_graph.nodes:
indranet_graph.add_node(ed['agB_name'],
ns=ed['agB_ns'],
id=ed['agB_id'])
# Add edges
ed.pop('agA_id')
ed.pop('agA_ns')
ed.pop('agB_id')
ed.pop('agB_ns')
if indranet_graph.is_multigraph():
# MultiDiGraph
indranet_graph.add_edge(ed['agA_name'], ed['agB_name'],
**ed)
else:
# DiGraph
u = ed.pop('agA_name')
v = ed.pop('agB_name')
# Check edge
if indranet_graph.has_edge(u, v):
indranet_graph.edges[(u,
v)]['statements'].append(ed)
else:
indranet_graph.add_edge(u,
v,
belief=1.0,
weight=1.0,
statements=[ed],
**z_sc_attrs)
logger.info('Loaded %d entity relations into dataframe' % entities)
indranet_graph.graph['node_by_uri'] = node_by_uri
indranet_graph.graph['node_by_ns_id'] = ns_id_to_nodename
indranet_graph.graph['edge_by_hash'] = hash_edge_dict
indranet_graph.graph['date'] = date
return indranet_graph