def selected(self, val):
skids = utils.eval_skids(val)
if not isinstance(skids, np.ndarray):
skids = np.array(skids)
neurons = self.neurons
logger.debug('{0} neurons selected ({1} previously)'.format(
len(skids), len(self.selected)))
# First un-highlight neurons no more selected
for s in [s for s in self.__selected if s not in set(skids)]:
for v in neurons[s]:
if isinstance(v, vp.scene.visuals.Mesh):
v.color = v._stored_color
else:
v.set_data(color=v._stored_color)
# Highlight new additions
for s in skids:
if s not in self.__selected:
for v in neurons[s]:
# Keep track of old colour
v.unfreeze()
v._stored_color = v.color
v.freeze()
if isinstance(v, vp.scene.visuals.Mesh):
v.color = self.highlight_color
else:
v.set_data(color=self.highlight_color)
self.__selected = skids
self.update_legend()
def hide_neurons(self, n):
""" Hide given neuron(s). """
skids = utils.eval_skids(n)
neurons = self.neurons
for s in skids:
for v in neurons[s]:
if v.visible:
v.visible = False
self.update_legend()
def toggle_neurons(self, n):
""" Toggle neuron(s) visibility. """
n = utils._make_iterable(n)
if False not in [isinstance(u, uuid.UUID) for u in n]:
obj = self._neuron_obj
else:
n = utils.eval_skids(n)
obj = self.neurons
for s in n:
for v in obj[s]:
v.visible = v.visible == False
self.update_legend()
def toggle_select(self, n):
""" Toggle selected of given neuron. """
skids = utils.eval_skids(n)
neurons = self.neurons
for s in skids:
if self.selected != s:
self.selected = s
for v in neurons[s]:
self._selected_color = v.color
v.set_data(color=self.highlight_color)
else:
self.selected = None
for v in neurons[s]:
v.set_data(color=self._selected_color)
self.update_legend()
def unhide_neurons(self, n=None, check_alpha=False):
""" Unhide given neuron(s). Use ``n`` to unhide specific neurons. """
if not isinstance(n, type(None)):
skids = utils.eval_skids(n)
else:
skids = list(self.neurons.keys())
neurons = self.neurons
for s in skids:
for v in neurons[s]:
if not v.visible:
v.visible = True
if check_alpha:
c = list(mcl.to_rgba(neurons[s][0].color))
if c[3] != 1:
c[3] = 1
self.set_colors({s: c})
self.update_legend()
def select(self, x, *args):
""" Select given neurons.
Parameters
----------
x : list of skeleton IDs | CatmaidNeuron/List | pd Dataframe
Returns
-------
:class:`pymaid.b3d.object_list` : containing requested neurons
Examples
--------
>>> selection = handler.select( [123456,7890] )
>>> # Get only connectors
>>> cn = selection.connectors
>>> # Hide everything else
>>> cn.hide_others()
>>> # Change color of presynapses
>>> selection.presynapses.color( 0, 1, 0 )
"""
skids = utils.eval_skids(x)
if not skids:
logger.error('No skids found.')
names = []
for ob in bpy.data.objects:
ob.select = False
if 'skeleton_id' in ob:
if ob['skeleton_id'] in skids:
ob.select = True
names.append(ob.name)
return object_list(names, handler=self)
def get_time_invested(x, remote_instance=None, minimum_actions=10,
treenodes=True, connectors=True, mode='SUM',
max_inactive_time=3, start_date=None, end_date=None):
""" Takes a list of neurons and calculates the time individual users
have spent working on this set of neurons.
Parameters
----------
x
Which neurons to check. Can be either:
1. skeleton IDs (int or str)
2. neuron name (str, must be exact match)
3. annotation: e.g. 'annotation:PN right'
4. CatmaidNeuron or CatmaidNeuronList object
If you pass a CatmaidNeuron/List, its data is used
calculate time invested. You can exploit this to get
time invested into a given compartment of a neurons,
e.g. by pruning it to a volume.
remote_instance : CatmaidInstance, optional
Either pass explicitly or define globally.
minimum_actions : int, optional
Minimum number of actions per minute to be counted as
active.
treenodes : bool, optional
If False, treenodes will not be taken into account
connectors : bool, optional
If False, connectors will not be taken into account
mode : 'SUM' | 'OVER_TIME' | 'ACTIONS', optional
(1) 'SUM' will return total time invested (in minutes)
per user.
(2) 'OVER_TIME' will return minutes invested/day over
time.
(3) 'ACTIONS' will return actions
(node/connectors placed/edited) per day.
max_inactive_time : int, optional
Maximal time inactive in minutes.
start_date : None | tuple | datetime.date, optional
end_date : None | tuple | datetime.date, optional
Restricts time invested to window. Applies to creation
but not edition time!
Returns
-------
pandas.DataFrame
If ``mode='SUM'``, values represent minutes invested.
>>> df
... total creation edition review
... user1
... user2
If ``mode='OVER_TIME'`` or ``mode='ACTIONS'``:
>>> df
... date date date ...
... user1
... user2
For `OVER_TIME`, values respresent minutes invested on that day. For
`ACTIONS`, values represent actions (creation, edition, review) on that
day.
Important
---------
Creation/Edition/Review times can overlap! This is why total time spent
is not just creation + edition + review.
Please note that this does currently not take placement of
pre-/postsynaptic nodes into account!
Be aware of the ``minimum_actions`` parameter: at low settings even
a single actions (e.g. connecting a node) will add considerably to time
invested. To keep total reconstruction time comparable to what Catmaid
calculates, you should consider about 10 actions/minute (= a click every
6 seconds) and ``max_inactive_time`` of 3 mins.
CATMAID gives reconstruction time across all users. Here, we calculate
the time spent tracing for individuals. This may lead to a discrepancy
between sum of time invested over of all users from this function vs.
CATMAID's reconstruction time.
Examples
--------
Plot pie chart of contributions per user using Plotly. This example
assumes that you have already imported and set up pymaid.
>>> import plotly
>>> stats = pymaid.get_time_invested(skids, remote_instance)
>>> # Use plotly to generate pie chart
>>> fig = {"data": [{"values": stats.total.tolist(),
... "labels": stats.user.tolist(), "type" : "pie" }]}
>>> plotly.offline.plot(fig)
Plot reconstruction efforts over time:
>>> stats = pymaid.get_time_invested(skids, mode='OVER_TIME')
>>> # Plot time invested over time
>>> stats.T.plot()
>>> # Plot cumulative time invested over time
>>> stats.T.cumsum(axis=0).plot()
>>> # Filter for major contributors
>>> stats[stats.sum(axis=1) > 20].T.cumsum(axis=0).plot()
"""
def _extract_timestamps(ts, desc='Calc'):
grouped = ts.set_index('timestamp',
drop=False).groupby(['user',
pd.Grouper(freq=bin_width)]).count() >= minimum_actions
temp_stats = {}
for u in config.tqdm(set(ts.user.unique()) & set(relevant_users),
desc=desc, disable=config.pbar_hide, leave=False):
temp_stats[u] = sum(grouped.loc[u].values)[0] * interval
return temp_stats
if mode not in ['SUM', 'OVER_TIME', 'ACTIONS']:
raise ValueError('Unknown mode "%s"' % str(mode))
remote_instance = utils._eval_remote_instance(remote_instance)
skids = utils.eval_skids(x, remote_instance)
# Maximal inactive time is simply translated into binning
# We need this later for pandas.TimeGrouper() anyway
interval = max_inactive_time
bin_width = '%iMin' % interval
# Update minimum_actions to reflect actions/interval instead of actions/minute
minimum_actions *= interval
user_list = fetch.get_user_list(remote_instance).set_index('id')
if not isinstance(x, (core.CatmaidNeuron, core.CatmaidNeuronList)):
x = fetch.get_neuron(skids, remote_instance=remote_instance)
if isinstance(x, core.CatmaidNeuron):
skdata = core.CatmaidNeuronList(x)
elif isinstance(x, core.CatmaidNeuronList):
skdata = x
if not isinstance(end_date, (datetime.date, type(None))):
end_date = datetime.date(*end_date)
if not isinstance(start_date, (datetime.date, type(None))):
start_date = datetime.date(*start_date)
# Extract connector and node IDs
node_ids = []
connector_ids = []
for n in skdata.itertuples():
if treenodes:
node_ids += n.nodes.treenode_id.tolist()
if connectors:
connector_ids += n.connectors.connector_id.tolist()
# Get node details
node_details = fetch.get_node_details(
node_ids + connector_ids, remote_instance=remote_instance)
# Get details for links
link_details = fetch.get_connector_links(skdata)
# link_details contains all links. We have to subset this to existing
# connectors in case the input neurons have been pruned
link_details = link_details[link_details.connector_id.isin(connector_ids)]
# Remove timestamps outside of date range (if provided)
if start_date:
node_details = node_details[node_details.creation_time >= np.datetime64(
start_date)]
link_details = link_details[link_details.creation_time >= np.datetime64(
start_date)]
if end_date:
node_details = node_details[node_details.creation_time <= np.datetime64(
end_date)]
link_details = link_details[link_details.creation_time <= np.datetime64(
end_date)]
# Dataframe for creation (i.e. the actual generation of the nodes)
creation_timestamps = np.append(node_details[['creator', 'creation_time']].values,
link_details[['creator_id', 'creation_time']].values,
axis=0)
creation_timestamps = pd.DataFrame(creation_timestamps,
columns=['user', 'timestamp'])
# Dataframe for edition times - can't use links as there is no editor
edition_timestamps = node_details[['editor', 'edition_time']]
edition_timestamps.columns = ['user', 'timestamp']
# Generate dataframe for reviews
reviewers = [u for l in node_details.reviewers.values for u in l]
timestamps = [ts for l in node_details.review_times.values for ts in l]
review_timestamps = pd.DataFrame([[u, ts] for u, ts in zip(
reviewers, timestamps)], columns=['user', 'timestamp'])
# Merge all timestamps
all_timestamps = pd.concat(
[creation_timestamps, edition_timestamps, review_timestamps], axis=0)
all_timestamps.sort_values('timestamp', inplace=True)
relevant_users = all_timestamps.groupby('user').count()
relevant_users = relevant_users[relevant_users.timestamp >= minimum_actions].index.values
if mode == 'SUM':
stats = {
'total': {u: 0 for u in relevant_users},
'creation': {u: 0 for u in relevant_users},
'edition': {u: 0 for u in relevant_users},
'review': {u: 0 for u in relevant_users}
}
stats['total'].update(_extract_timestamps(all_timestamps,
desc='Calc total'))
stats['creation'].update(_extract_timestamps(creation_timestamps,
desc='Calc creation'))
stats['edition'].update(_extract_timestamps(edition_timestamps,
desc='Calc edition'))
stats['review'].update(_extract_timestamps(review_timestamps,
desc='Calc review'))
return pd.DataFrame([[user_list.loc[u, 'login'],
stats['total'][u],
stats['creation'][u],
stats['edition'][u],
stats['review'][u]] for u in relevant_users],
columns=['user', 'total', 'creation', 'edition', 'review']).sort_values('total', ascending=False).reset_index(drop=True).set_index('user')
elif mode == 'ACTIONS':
all_ts = all_timestamps.set_index('timestamp', drop=False).timestamp.groupby(
pd.Grouper(freq='1d')).count().to_frame()
all_ts.columns = ['all_users']
all_ts = all_ts.T
# Get total time spent
for u in config.tqdm(all_timestamps.user.unique(), desc='Calc. total', disable=config.pbar_hide, leave=False):
this_ts = all_timestamps[all_timestamps.user == u].set_index(
'timestamp', drop=False).timestamp.groupby(pd.Grouper(freq='1d')).count().to_frame()
this_ts.columns = [user_list.loc[u, 'login']]
all_ts = pd.concat([all_ts, this_ts.T])
return all_ts.fillna(0)
elif mode == 'OVER_TIME':
# First count all minutes with minimum number of actions
minutes_counting = (all_timestamps.set_index('timestamp', drop=False).timestamp.groupby(
pd.Grouper(freq=bin_width)).count().to_frame() > minimum_actions)
# Then remove the minutes that have less than minimum actions
minutes_counting = minutes_counting[minutes_counting.timestamp == True]
# Now group by hour
all_ts = minutes_counting.groupby(pd.Grouper(freq='1d')).count()
all_ts.columns = ['all_users']
all_ts = all_ts.T
# Get total time spent
for u in config.tqdm(all_timestamps.user.unique(), desc='Calc. total', disable=config.pbar_hide, leave=False):
minutes_counting = (all_timestamps[all_timestamps.user == u].set_index(
'timestamp', drop=False).timestamp.groupby(pd.Grouper(freq=bin_width)).count().to_frame() > minimum_actions)
minutes_counting = minutes_counting[minutes_counting.timestamp == True]
this_ts = minutes_counting.groupby(pd.Grouper(freq='1d')).count()
this_ts.columns = [user_list.loc[u, 'login']]
all_ts = pd.concat([all_ts, this_ts.T])
all_ts.fillna(0, inplace=True)
return all_ts
def get_user_contributions(x, teams=None, remote_instance=None):
""" Takes a list of neurons and returns nodes and synapses contributed
by each user.
Notes
-----
This is essentially a wrapper for :func:`pymaid.get_contributor_statistics`
- if you are also interested in e.g. construction time, review time, etc.
you may want to consider using :func:`~pymaid.get_contributor_statistics`
instead.
Parameters
----------
x
Which neurons to check. Can be either:
1. skeleton IDs (int or str)
2. neuron name (str, must be exact match)
3. annotation: e.g. 'annotation:PN right'
4. CatmaidNeuron or CatmaidNeuronList object
teams dict, optional
Teams to group contributions for. Users must be logins::
{'teamA': ['user1', 'user2'], 'team2': ['user3'], ...]}
Users not part of any team, will be grouped as team
``'others'``.
remote_instance : Catmaid Instance, optional
Either pass explicitly or define globally.
Returns
-------
pandas.DataFrame
DataFrame in which each row represents a user
>>> df
... user nodes presynapses postsynapses nodes_reviewed
... 0
... 1
Examples
--------
>>> import matplotlib.pyplot as plt
>>> # Get contributors for a single neuron
>>> cont = pymaid.get_user_contributions(2333007)
>>> # Get top 10 (by node contribution)
>>> top10 = cont.iloc[:10].set_index('user')
>>> # Plot as bar chart
>>> ax = top10.plot(kind='bar')
>>> plt.show()
>>> # Plot relative contributions
>>> cont = pymaid.get_user_contributions(2333007)
>>> cont = cont.set_index('user')
>>> # Normalise
>>> cont_rel = cont / cont.sum(axis=0).values
>>> # Plot contributors with >5% node contributions
>>> ax = cont_rel[cont_rel.nodes > .05].plot(kind='bar')
>>> plt.show()
See Also
--------
:func:`~pymaid.get_contributor_statistics`
Gives you more basic info on neurons of interest
such as total reconstruction/review time.
"""
if not isinstance(teams, type(None)):
# Prepare teams
if not isinstance(teams, dict):
raise TypeError('Expected teams of type dict, got {}'.format(type(teams)))
for t in teams:
if not isinstance(teams[t], list):
raise TypeError('Teams need to list of user logins, got {}'.format(type(teams[t])))
# Turn teams into a login -> team dict
teams = {u : t for t in teams for u in teams[t]}
remote_instance = utils._eval_remote_instance(remote_instance)
skids = utils.eval_skids(x, remote_instance)
cont = fetch.get_contributor_statistics(
skids, remote_instance, separate=False)
all_users = set(list(cont.node_contributors.keys(
)) + list(cont.pre_contributors.keys()) + list(cont.post_contributors.keys()))
stats = {
'nodes': {u: 0 for u in all_users},
'presynapses': {u: 0 for u in all_users},
'postsynapses': {u: 0 for u in all_users},
'nodes_reviewed': {u: 0 for u in all_users}
}
for u in cont.node_contributors:
stats['nodes'][u] = cont.node_contributors[u]
for u in cont.pre_contributors:
stats['presynapses'][u] = cont.pre_contributors[u]
for u in cont.post_contributors:
stats['postsynapses'][u] = cont.post_contributors[u]
for u in cont.review_contributors:
stats['nodes_reviewed'][u] = cont.review_contributors[u]
stats = pd.DataFrame([[u, stats['nodes'][u],
stats['presynapses'][u],
stats['postsynapses'][u],
stats['nodes_reviewed'][u]] for u in all_users],
columns=['user', 'nodes', 'presynapses',
'postsynapses', 'nodes_reviewed']
).sort_values('nodes', ascending=False).reset_index(drop=True)
if isinstance(teams, type(None)):
return stats
stats['team'] = [teams.get(u, 'others') for u in stats.user.values]
return stats.groupby('team').sum()
def adjacency_matrix(s,
t=None,
remote_instance=None,
source_grp={},
target_grp={},
syn_threshold=None,
syn_cutoff=None,
use_connectors=False):
""" Generate adjacency matrix for synaptic connections between sets of
neurons. Directional: sources = rows, targets = columns.
Parameters
----------
s
Source neurons as single or list of either:
1. skeleton IDs (int or str)
2. neuron name (str, exact match)
3. annotation: e.g. 'annotation:PN right'
4. CatmaidNeuron or CatmaidNeuronList object
t
Optional. Target neurons as single or list of either:
1. skeleton IDs (int or str)
2. neuron name (str, exact match)
3. annotation: e.g. ``'annotation:PN right'`
4. CatmaidNeuron or CatmaidNeuronList object
If not provided, ``source neurons = target neurons``.
remote_instance : CATMAID instance, optional
syn_cutoff : int, optional
If set, will cut off connections above given value.
syn_threshold : int, optional
If set, will ignore connections with less synapses.
source_grp : dict, optional
Use to collapse sources into groups. Can be either:
1. ``{group1: [neuron1, neuron2, ... ], ..}``
2. ``{neuron1: group1, neuron2 : group2, ..}``
``syn_cutoff`` and ``syn_threshold`` are applied
BEFORE grouping!
target_grp : dict, optional
See ``source_grp`` for possible formats.
use_connectors : bool, optional
If True AND ``s`` or ``t`` are ``CatmaidNeuron/List``,
restrict adjacency matrix to their connectors. Use
if e.g. you are using pruned neurons. **Important**:
This does not work if you have multiple fragments per
neuron!
Returns
-------
matrix : ``pandas.Dataframe``
See Also
--------
:func:`~pymaid.group_matrix`
More fine-grained control over matrix grouping.
:func:`~pymaid.adjacency_from_connectors`
Use this function if you are working with multiple fragments
per neuron.
Examples
--------
Generate and plot a adjacency matrix:
>>> import seaborn as sns
>>> import matplotlib.pyplot as plt
>>> import pymaid
>>> rm = pymaid.CatmaidInstance(url, user, pw,, token)
>>> neurons = pymaid.get_neurons('annotation:test')
>>> mat = pymaid.adjacency_matrix( neurons )
>>> g = sns.heatmap(adj_mat, square=True)
>>> g.set_yticklabels(g.get_yticklabels(), rotation = 0, fontsize = 7)
>>> g.set_xticklabels(g.get_xticklabels(), rotation = 90, fontsize = 7)
>>> plt.show()
Cut neurons into axon dendrites and compare their connectivity:
>>> # Get a set of neurons
>>> nl = pymaid.get_neurons('annnotation:type_16_candidates')
>>> # Split into axon dendrite by using a tag
>>> nl.reroot(nl.soma)
>>> nl_axon = nl.prune_proximal_to('axon', inplace=False)
>>> nl_dend = nl.prune_distal_to('axon', inplace=False)
>>> # Get a list of the downstream partners
>>> cn_table = pymaid.get_partners(nl)
>>> ds_partners = cn_table[ cn_table.relation == 'downstream' ]
>>> # Take the top 10 downstream partners
>>> top_ds = ds_partners.iloc[:10].skeleton_id.values
>>> # Generate separate adjacency matrices for axon and dendrites
>>> adj_axon = pymaid.adjacency_matrix(nl_axon, top_ds, use_connectors=True )
>>> adj_dend = pymaid.adjacency_matrix(nl_dend, top_ds, use_connectors=True )
>>> # Rename rows and merge dataframes
>>> adj_axon.index += '_axon'
>>> adj_dend.index += '_dendrite'
>>> adj_merged = pd.concat([adj_axon, adj_dend], axis=0)
>>> # Plot heatmap using seaborn
>>> ax = sns.heatmap(adj_merged)
>>> plt.show()
"""
remote_instance = utils._eval_remote_instance(remote_instance)
if t is None:
t = s
neuronsA = utils.eval_skids(s, remote_instance=remote_instance)
neuronsB = utils.eval_skids(t, remote_instance=remote_instance)
# Make sure neurons are integers
neurons = list(set([int(n) for n in (neuronsA + neuronsB)]))
neuronsA = [int(n) for n in neuronsA]
neuronsB = [int(n) for n in neuronsB]
# Make sure neurons are unique
neuronsA = sorted(set(neuronsA), key=neuronsA.index)
neuronsB = sorted(set(neuronsB), key=neuronsB.index)
logger.info('Retrieving and filtering connectivity...')
if use_connectors and (
isinstance(s, (core.CatmaidNeuron, core.CatmaidNeuronList))
or isinstance(t, (core.CatmaidNeuron, core.CatmaidNeuronList))):
edges = _edges_from_connectors(s, t, remote_instance=remote_instance)
else:
edges = fetch.get_edges(neurons, remote_instance=remote_instance)
# Turn into a adjacency matrix
matrix = edges.pivot(values='weight',
columns='target_skid',
index='source_skid').fillna(0)
# Filter to actual sources and targets
matrix = matrix.reindex(neuronsA, columns=neuronsB, fill_value=0)
# Apply cutoff and threshold
matrix = matrix.clip(upper=syn_cutoff)
if syn_threshold:
matrix[matrix < syn_threshold] = 0
matrix.datatype = 'adjacency_matrix'
if source_grp or target_grp:
matrix = group_matrix(matrix,
source_grp,
target_grp,
drop_ungrouped=False)
logger.info('Finished!')
return matrix
def adjacency_from_connectors(source, target=None, remote_instance=None):
""" Regenerates adjacency matrices from neurons' connectors.
Notes
-----
This function creates an adjacency matrix from scratch using just the
neurons' connectors. This function is able to deal with non-unique
skeleton IDs (most other functions are not). Use it e.g. when you
split neurons into multiple fragments.
Parameters
----------
source,target : skeleton IDs | CatmaidNeuron | CatmaidNeuronList
Neuron(s) for which to generate adjacency matrix.
If ``target==None``, will use ``target=source``.
Returns
-------
pandas.DataFrame
Matrix holding possible synaptic contacts. Sources are rows,
targets are columns. Labels are skeleton IDs. Order is preserved.
>>> df
target1 target2 target3 ...
source1 5 1 0
source2 10 20 5
source3 4 3 15
...
See Also
--------
:func:`~pymaid.adjacency_matrix`
If you are working with "intact" neurons. Much faster!
:func:`~pymaid.filter_connectivity`
Use this function if you have only a single fragment per neuron
(e.g. just the axon). Also way faster.
Examples
--------
>>> # Fetch some neurons
>>> x = pymaid.get_neuron('annotation:PD2a1/b1')
>>> # Split into axon / dendrites
>>> x.reroot(x.soma)
>>> split = pymaid.split_axon_dendrite(x)
>>> # Regenerate all-by-all adjacency matrix
>>> adj = pymaid.adjacency_from_connectors(split)
>>> # Skeleton IDs are non-unique but column/row order = input order:
>>> # in this example, the first occurrence is axon, the second dendrites
>>> adj.head()
"""
remote_instance = utils._eval_remote_instance(remote_instance)
if not isinstance(source, (core.CatmaidNeuron, core.CatmaidNeuronList)):
skids = utils.eval_skids(source)
source = fetch.get_neuron(skids, remote_instance=remote_instance)
if isinstance(target, type(None)):
target = source
elif not isinstance(target, (core.CatmaidNeuron, core.CatmaidNeuronList)):
skids = utils.eval_skids(target)
target = fetch.get_neuron(skids, remote_instance=remote_instance)
if isinstance(source, core.CatmaidNeuron):
source = core.CatmaidNeuronList(source)
if isinstance(target, core.CatmaidNeuron):
target = core.CatmaidNeuronList(target)
# Generate empty adjacency matrix
adj = np.zeros((len(source), len(target)))
# Get connector details for all neurons
all_cn = list(
set(
np.append(source.connectors.connector_id.values,
target.connectors.connector_id.values)))
cn_details = fetch.get_connector_details(all_cn)
# Now go over all source neurons and process connections
for i, s in enumerate(
config.tqdm(source,
desc='Processing',
disable=config.pbar_hide,
leave=config.pbar_leave)):
# Get all connectors presynaptic for this source
this_cn = cn_details[
(cn_details.presynaptic_to == int(s.skeleton_id))
& (cn_details.connector_id.isin(s.connectors.connector_id))]
# Go over all target neurons
for k, t in enumerate(target):
t_tn = set(t.nodes.treenode_id.values)
t_post = t.postsynapses.connector_id.values
# Extract number of connections from source to this target
this_t = this_cn[this_cn.connector_id.isin(t_post)]
# Now figure out how many links are between this connector and
# the target
n_links = sum([
len(t_tn & set(r.postsynaptic_to_node))
for r in this_t.itertuples()
])
adj[i][k] = n_links
return pd.DataFrame(adj,
index=source.skeleton_id,
columns=target.skeleton_id)
def network2nx(x, remote_instance=None, threshold=1):
""" Generates NetworkX graph for neuron connectivity.
Parameters
----------
x
Catmaid Neurons as:
1. list of skeleton IDs (int or str)
2. list of neuron names (str, exact match)
3. annotation(s): e.g. 'annotation:PN right'
4. CatmaidNeuronList object
5. Adjacency matrix (pd.DataFrame, rows=sources,
columns=targets)
remote_instance : CATMAID instance, optional
Either pass directly to function or define globally
as 'remote_instance'.
threshold : int, optional
Connections weaker than this will be excluded.
Returns
-------
networkx.DiGraph
NetworkX representation of the network.
Examples
--------
>>> import matplotlib.pyplot as plt
>>> import networkx as nx
>>> import numpy as np
>>> g = pymaid.network2graph('annotation:large network')
>>> # Plot with default settings
>>> nx.draw(g)
>>> plt.show()
>>> # Plot with neuron names
>>> labels = nx.get_node_attributes(g, 'neuron_name')
>>> nx.draw(g, labels=labels, with_labels=True)
>>> plt.show()
>>> # Plot with layout
>>> layout = nx.circular_layout(g)
>>> nx.draw(g, pos=layout)
>>> plt.show()
>>> # Plot with edge weights
>>> nx.draw_networkx_nodes(g, pos=layout)
>>> weight = np.array(list(nx.get_edge_attributes(g, 'weight').values()))
>>> nx.draw_networkx_edges(g, pos=layout, width=weight/50)
>>> plt.show()
"""
if isinstance(x, (core.CatmaidNeuronList, list, np.ndarray, str)):
remote_instance = utils._eval_remote_instance(remote_instance)
skids = utils.eval_skids(x, remote_instance=remote_instance)
# Fetch edges
edges = fetch.get_edges(skids, remote_instance=remote_instance)
# Reformat into networkx format
edges = [[
str(e.source_skid),
str(e.target_skid), {
'weight': e.weight
}
] for e in edges[edges.weight >= threshold].itertuples()]
elif isinstance(x, pd.DataFrame):
# We have to account for the fact that some might not be skids
skids = []
for s in list(set(x.columns.tolist() + x.index.tolist())):
try:
skids.append(int(s))
except BaseException:
pass
# Generate edge list
edges = [[str(s), str(t), {
'weight': x.loc[s, t]
}] for s in x.index.values for t in x.columns.values
if x.loc[s, t] >= threshold]
else:
raise ValueError('Unable to process data of type "{0}"'.format(
type(x)))
# Generate node dictionary
names = fetch.get_names(skids, remote_instance=remote_instance)
nodes = [[str(s), {'neuron_name': names.get(s, s)}] for s in skids]
# Generate graph and assign custom properties
g = nx.DiGraph()
g.add_nodes_from(nodes)
g.add_edges_from(edges)
return g
def network2igraph(x, remote_instance=None, threshold=1):
""" Generates iGraph graph for neuron connectivity. Requires iGraph to be
installed.
Parameters
----------
x
Catmaid Neurons as:
1. list of skeleton IDs (int or str)
2. list of neuron names (str, exact match)
3. annotation(s): e.g. 'annotation:PN right'
4. CatmaidNeuronList object
5. Adjacency matrix (pd.DataFrame, rows=sources,
columns=targets)
remote_instance : CATMAID instance, optional
Either pass directly to function or define globally
as 'remote_instance'.
threshold : int, optional
Connections weaker than this will be excluded .
Returns
-------
igraph.Graph(directed=True)
NetworkX representation of the network.
Examples
--------
>>> import pymaid
>>> import igraph
>>> g = pymaid.network2igraph('annotation:large network', remote_instance=rm)
>>> # Plot graph
>>> igraph.plot(g)
>>> # Plot with edge width
>>> igraph.plot(g, **{'edge_width': [ w/10 for w in g.es['weight'] ] })
>>> # Plot with edge label
>>> igraph.plot(g, **{'edge_label': g.es['weight'] })
>>> # Save as graphml to import into e.g. Cytoscape
>>> g.save('graph.graphml')
"""
if igraph is None:
raise ImportError('igraph must be installed to use this function.')
if isinstance(x, (core.CatmaidNeuronList, list, np.ndarray, str)):
remote_instance = utils._eval_remote_instance(remote_instance)
skids = utils.eval_skids(x, remote_instance=remote_instance)
indices = {int(s): i for i, s in enumerate(skids)}
# Fetch edges
edges = fetch.get_edges(skids, remote_instance=remote_instance)
# Reformat into igraph format
edges_by_index = [[
indices[e.source_skid], indices[e.target_skid]
] for e in edges[edges.weight >= threshold].itertuples()]
weight = edges[edges.weight >= threshold].weight.tolist()
elif isinstance(x, pd.DataFrame):
skids = list(set(x.columns.tolist() + x.index.tolist()))
# Generate edge list
edges = [[i, j] for i in x.index.tolist() for j in x.columns.tolist()
if x.loc[i, j] >= threshold]
edges_by_index = [[skids.index(e[0]),
skids.index(e[1])] for e in edges]
weight = [
x.loc[i, j] for i in range(x.shape[0]) for j in range(x.shape[1])
if x.loc[i, j] >= threshold
]
else:
raise ValueError('Unable to process data of type "{0}"'.format(
type(x)))
# Generate igraph and assign custom properties
g = igraph.Graph(directed=True)
g.add_vertices(len(skids))
g.add_edges(edges_by_index)
g.vs['node_id'] = skids
# g.vs['neuron_name'] = g.vs['label'] = neuron_names
g.es['weight'] = weight
return g
def take_snapshot(x,
skeleton_data=True,
cn_table=False,
node_details=False,
adjacency_matrix=True,
remote_instance=None,
cn_details=True,
annotations=False):
""" Take a snapshot of CATMAID data associated with a set of neurons.
Important
---------
If you pass Catmaidneuron/List that have been modified (e.g. pruned),
other data (e.g. connectivity, etc) will be subset as well if applicable.
If your CatmaidNeuron/List is still naive, you might want to just pass the
skeleton ID(s) to speed things up.
Parameters
----------
x : skeleton IDs | CatmaidNeuron/List
Neurons for which to retrieve data.
skeleton_data : bool, optional
Include 3D skeleton data.
cn_table : bool, optional
Include connectivity table. Covers all neurons
connected to input neurons.
node_details : bool, optional
Include treenode and connector details.
adjacency_matrix : bool, optional
Include adjacency matrix covering the input neurons.
cn_details : bool, optional
Include connector details.
annotations : bool, optional
Include neuron annotations.
remote_instance : Catmaid Instance, optional
Either pass explicitly or define globally. Will
obviously not be added to the snapshot!
Returns
-------
pandas Series
Examples
--------
See Also
--------
:func:`~pymaid.load_snapshot`
Use to load a snapshot file.
"""
remote_instance = utils._eval_remote_instance(remote_instance)
skids = utils.eval_skids(x, remote_instance)
snapshot = pd.Series()
# Add Coordinates Universal Time
snapshot['utc_date'] = datetime.datetime.utcnow()
# Add pymaid version
snapshot['pymaid_version'] = init.__version__
# Add skeleton data
if skeleton_data:
if not isinstance(x, (core.CatmaidNeuronList, core.CatmaidNeuron)):
skdata = fetch.get_neurons(skids, remote_instance=remote_instance)
else:
skdata = x
if isinstance(skdata, core.CatmaidNeuron):
skdata = core.CatmaidNeuronList(skdata)
snapshot['skeleton_data'] = skdata.to_dataframe()
# Add connectivity table
if cn_table:
if isinstance(x, (core.CatmaidNeuron, core.CatmaidNeuronList)):
snapshot['cn_table'] = connectivity.cn_table_from_connectors(
x, remote_instance=remote_instance)
else:
# Add connectivity table
snapshot['cn_table'] = fetch.get_partners(
x, remote_instance=remote_instance)
# Add connectivity table
if node_details:
snapshot['treenode_details'] = fetch.get_node_details(
skdata.nodes.treenode_id.values, remote_instance=remote_instance)
snapshot['connector_details'] = fetch.get_node_details(
skdata.connectors.connector_id.values,
remote_instance=remote_instance)
# Add adjacency matrix
if adjacency_matrix:
if isinstance(x, (core.CatmaidNeuron, core.CatmaidNeuronList)):
snapshot[
'adjacency_matrix'] = connectivity.adjacency_from_connectors(
x, remote_instance=remote_instance)
else:
# Add connectivity table
snapshot['adjacency_matrix'] = connectivity.adjacency_matrix(
x, remote_instance=remote_instance)
# Add annotations
if annotations:
snapshot['annotations'] = fetch.get_annotations(skids)
# Add connector details
if cn_details:
snapshot['cn_details'] = fetch.get_connector_details(skdata)
return snapshot
def watch_network(x,
sleep=3,
n_circles=1,
min_pre=2,
min_post=2,
layout=None,
remote_instance=None,
verbose=True):
""" Loads and **continuously updates** a network into Cytoscape. Use
CTRL-C to stop.
Parameters
----------
x : skeleton IDs | CatmaidNeuron/List
Seed neurons to keep track of.
sleep : int | None, optional
Time in seconds to sleep after each update.
n_circles : int, optional
Number of circles around seed neurons to include in
the network. See also :func:`pymaid.get_nth_partners`.
Set to ``None | 0 | False`` to only update
seed nodes.
min_pre/min_post : int, optional
Synapse threshold to apply to ``n_circles``.
Set to -1 to not get any pre-/post synaptic partners.
Please note: as long as there is a single
above-threshold connection, a neuron will be included.
This does not remove other, sub-threshold connections.
layout : str | None, optional
Name of a Cytoscape layout. If provided, will update
the network's layout on every change.
remote_instance : CatmaidInstance, optional
verbose : bool, optional
If True, will log changes made to the network.
Returns
-------
Nothing
Examples
--------
>>> import pymaid
>>> import pymaid.cytoscape as cytomaid
>>> rm = pymaid.CatmaidInstance('server_url', 'http_user',
... 'http_pw', 'auth_token')
>>> # Don't forget to start Cytoscape!
>>> cytomaid.watch_network('annotation:glomerulus DA1', min_pre=5,
... min_post=-1, sleep=5)
>>> # Use CTRL-C to stop the loop
"""
cy = get_client()
remote_instance = utils._eval_remote_instance(remote_instance)
sleep = 0 if not sleep else sleep
x = utils.eval_skids(x, remote_instance=remote_instance)
# Generate the initial network
if n_circles:
to_add = fetch.get_nth_partners(
x,
n_circles=n_circles,
min_pre=min_pre,
min_post=min_post,
remote_instance=remote_instance).skeleton_id
else:
to_add = []
g = graph.network2nx(np.concatenate([x, to_add]).astype(int),
remote_instance=remote_instance)
network = generate_network(g,
clear_session=True,
apply_style=False,
layout=layout)
if layout:
cy.layout.apply(name=layout, network=network)
logger.info('Watching network. Use CTRL-C to stop.')
if remote_instance.caching:
logger.warning('Caching disabled.')
remote_instance.caching = False
utils.set_loggers('WARNING')
while True:
if n_circles:
to_add = fetch.get_nth_partners(
x,
n_circles=n_circles,
min_pre=min_pre,
min_post=min_post,
remote_instance=remote_instance).skeleton_id
else:
to_add = []
g = graph.network2nx(np.concatenate([x, to_add]).astype(int),
remote_instance=remote_instance)
# Add nodes that came in new
ntable = network.get_node_table()
nodes_to_add = [s for s in g.nodes if s not in ntable.id.values]
if nodes_to_add:
network.add_nodes(nodes_to_add)
# Update neuron names
ntable = network.get_node_table()
names = ntable.set_index('name').neuron_name.to_dict()
names.update({s: g.nodes[s]['neuron_name'] for s in g.nodes})
ntable['id'] = ntable.name
ntable['neuron_name'] = ntable.name.map(names)
network.update_node_table(ntable,
data_key_col='name',
network_key_col='name')
# Remove nodes that do not exist anymore
ntable = network.get_node_table()
nodes_to_remove = ntable[~ntable['id'].isin(g.nodes)]
if not nodes_to_remove.empty:
for v in nodes_to_remove.SUID.values:
network.delete_node(v)
# Remove edges
etable = network.get_edge_table()
edges_removed = 0
for e in etable.itertuples():
if (e.source, e.target) not in g.edges:
edges_removed += 1
network.delete_edge(e.SUID)
# Add edges
etable = network.get_edge_table()
edges = [(s, t)
for s, t in zip(etable.source.values, etable.target.values)]
skid_to_SUID = ntable.set_index('name').SUID.to_dict()
edges_to_add = []
for e in set(g.edges) - set(edges):
edges_to_add.append({
'source': skid_to_SUID[e[0]],
'target': skid_to_SUID[e[1]],
'interaction': None,
'directed': True
})
if edges_to_add:
network.add_edges(edges_to_add)
# Fix table and modify weights if applicable
etable = network.get_edge_table()
if not etable.loc[etable.source.isnull()].empty:
etable.loc[etable.source.isnull(), 'source'] = etable.loc[
etable.source.isnull(),
'name'].map(lambda x: x[:x.index('(') - 1])
etable.loc[etable.target.isnull(), 'target'] = etable.loc[
etable.target.isnull(),
'name'].map(lambda x: x[x.index(')') + 2:])
new_weights = [
g.edges[e]['weight'] for e in etable[['source', 'target']].values
]
weights_modified = [
new_w for new_w, old_w in zip(new_weights, etable.weight.values)
if new_w != old_w
]
etable['weight'] = new_weights
# For some reason, there os no official wrapper for this, so we have to get our hands dirty
network._CyNetwork__update_table('edge',
etable,
network_key_col='SUID',
data_key_col='SUID')
# If changes were made, give some feedback and/or change layout
if nodes_to_add or not nodes_to_remove.empty or edges_to_add or edges_removed or weights_modified:
if verbose:
logger.info(
'{} - nodes added/removed: {}/{}; edges added/removed/modified {}/{}/{}'
.format(
datetime.datetime.now(),
len(nodes_to_add),
len(nodes_to_remove),
len(edges_to_add),
edges_removed,
len(weights_modified),
))
if layout:
cy.layout.apply(name=layout, network=network)
# ZzzZzzzZ
time.sleep(sleep)