return catmaid.algorithms.population.distance.near_path_length(
nd,
na,
g1=nd.dendrites,
g2=na.dendrites,
distance=distance,
resample_distance=resample_distance)
except Exception as e:
return e
n_jobs = -1
output_file = '../../results/scripts/dendrite_near_path_lengths_%i_%i.csv' % (
int(distance), int(resample_distance))
s = catmaid.get_source('../../data/skeletons')
s._cache = None
pairs = []
if os.path.exists(pairs_fn):
print("Loading pairs from csv: %s" % pairs_fn)
with open(pairs_fn, 'r') as f:
for l in f:
if l.strip() != '':
pairs.append(map(int, l.strip().split(',')))
else:
raise Exception
sids = []
with open('skels2pull.csv', 'r') as f:
for l in f:
if l.strip() != '':
sids.append(l.strip())
'soma',
'myelinated',
'unmyelinated',
'root_myelinated',
'root_unmyelinated',
'ends',
'damage',
'axon',
'projection',
'backbone',
]
# connect to catmaid and get the source from the connection
# default to using environmental variables
c = catmaid.connect()
src = catmaid.get_source(c, ignore_none_skeletons=True)
# create logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
# creates two lists, one for error messages and one for url links
master_errors = []
master_urls = []
root_errors, root_urls = [], []
leaf_tag_errors, leaf_tag_urls = [], []
loop_errors, loop_urls = [], []
incorrect_tag_errors, incorrect_tag_urls = [], []
max_distance_errors, max_distance_urls = [], []
min_distance_errors, min_distance_urls = [], []
import catmaid
import numpy
import pickle
from time import strftime
import sys
filesource = '../../data/skeletons/'
# TODO transformfile should be set as environmental variable
# and an error thrown if not set
transformfile = './physical_section_affine_4x4.p'
resXYZ = numpy.array([18.85, 18.85, 60.])
src = catmaid.get_source(filesource)
try:
outfile = str(sys.argv[1])
except:
outfile = '../../results/exports/{}_130201zf142_' \
'ALLNODE_dump_RAW.txt'.format(strftime('%y%m%dT%H%M'))
with open(transformfile, 'r') as f:
transforms = pickle.load(f)
with open(outfile, 'w') as f:
for sid in src.skeleton_ids():
n = src.get_neuron(sid)
#if any([('blacklist' in anno) for anno in n.annotations]):
# continue
for nid in n.nodes:
if nid == n.root:
parent = 'root'
else:
help="Max number of attempts to fetch skeletons in case error encountered")
parser.add_argument(
'-n',
'--ignore_none_skeletons',
action='store_true',
help="Option to ignore fetching invalid skeletons with type None, "
"such as those which have been deleted or merged "
"since starting fetch.")
opts = parser.parse_args()
# create a skeleton source (which connects to catmaid)
# the server, user and project to use can be set by
# - environment variables (see README)
# - interactive command prompts
# - creating and passing in a connection (see catmaid.connect)
source = catmaid.get_source(ignore_none_skeletons=opts.ignore_none_skeletons)
# source is now a ServerSource that can fetch skeletons from catmaid
# check if only a subset of ids should be saved
sids = opts.sids
if opts.idfile is not None:
sids = [] # overwrite any ids provided on the command line
ext = os.path.splitext(opts.idfile)[1].lower()
if ext in ('.p', '.pickle', '.pkl'):
# assume these are pickled iterators
import cPickle as pickle
with open(opts.idfile, 'r') as f:
sids = pickle.load(f)
elif ext in ('.json', '.js'):
import json
def collect_project_data(source, conn=None, skels_list=None, except_anno=None):
if except_anno is not None:
print "Excluding all skeletons with annotation(s): {}".format(
except_anno)
print "Collecting Project Data..."
all_data = {}
numskels = 0
somareconstructed = 0
max_somapathlengths = {}
min_somapathlengths = {}
mean_somapathlengths = {}
median_somapathlengths = {}
max_leafpathlengths = {}
min_leafpathlengths = {}
mean_leafpathlengths = {}
median_leafpathlengths = {}
hassoma = {}
totalpaths = {}
allsomapathlengths, allleafpathlengths = [], []
s = catmaid.get_source(source)
if skels_list is not None:
skels = skels_list
else:
skels = s.skeleton_ids()
for sid in skels:
# Create empty variable for skipping a skel with x annotation
skip_skel = 0
if type(sid) == list:
sid = sid[0]
n = s.get_neuron(sid)
if except_anno is not None:
if len(except_anno) > 1:
for e in except_anno:
if any(a[0] == e for a in n.annotations):
skip_skel += 1
else:
if any(a[0] == except_anno[0] for a in n.annotations):
continue
# Skip this skeleton if it has any excluded annotations
if skip_skel != 0:
continue
totalpaths[sid] = total_pathlength(n)
if len(n.nodes.keys()) > nnodethres:
numskels += 1
if 'soma' in n.tags:
hassoma[sid] = 1
else:
hassoma[sid] = 0
if len(n.nodes) > 1:
rootpathlengths = [path_length(n, n.root, leaf)
for leaf in n.leaves]
if len(n.leaves) > 1:
leafpathlengths = [path_length(n, leaf1, leaf2)
for leaf1, leaf2 in combinations(
n.leaves, 2)]
else:
leafpathlengths = rootpathlengths
if hassoma[sid]:
somareconstructed += 1
max_somapathlengths[sid] = max(rootpathlengths)
min_somapathlengths[sid] = min(rootpathlengths)
mean_somapathlengths[sid] = numpy.mean(rootpathlengths)
median_somapathlengths[sid] = numpy.median(rootpathlengths)
allsomapathlengths.extend(rootpathlengths)
allleafpathlengths.extend(leafpathlengths)
max_leafpathlengths[sid] = max(leafpathlengths)
min_leafpathlengths[sid] = min(leafpathlengths)
mean_leafpathlengths[sid] = numpy.mean(leafpathlengths)
median_leafpathlengths[sid] = numpy.median(leafpathlengths)
project_pathlength = sum(totalpaths.values())
max_pathlength = max(totalpaths.values())
mean_pathlength = numpy.mean(totalpaths.values())
min_pathlength = min(totalpaths.values())
median_pathlength = numpy.median(totalpaths.values())
project_somata = somareconstructed
somapercent = float(project_somata)/float(numskels)
projdata = {'length traced': project_pathlength,
'number of somata': project_somata,
'number of reconstructions': numskels,
'percent with soma': somapercent,
'max length traced': max_pathlength,
'min length traced': min_pathlength,
'mean length traced': mean_pathlength,
'median length traced': median_pathlength,
'soma to leaf max': max(allsomapathlengths),
'soma to leaf min': min(allsomapathlengths),
'soma to leaf median': numpy.median(allsomapathlengths),
'soma to leaf mean': numpy.mean(allsomapathlengths),
'total pathlength of neurons with a soma': sum(allsomapathlengths),
'leaf to leaf max': max(allleafpathlengths),
'leaf to leaf min': min(allleafpathlengths),
'leaf to leaf median': numpy.median(allleafpathlengths),
'leaf to leaf mean': numpy.mean(allleafpathlengths)}
all_data['max_somapathlengths'] = max_somapathlengths
all_data['min_somapathlengths'] = min_somapathlengths
all_data['mean_somapathlengths'] = mean_somapathlengths
all_data['median_somapathlengths'] = median_somapathlengths
all_data['max_leafpathlengths'] = max_leafpathlengths
all_data['min_leafpathlengths'] = min_leafpathlengths
all_data['mean_leafpathlengths'] = mean_leafpathlengths
all_data['median_leafpathlengths'] = median_leafpathlengths
all_data['hassoma'] = hassoma
all_data['totalpaths'] = totalpaths
all_data['allsomapathlengths'] = allsomapathlengths
all_data['allleafpathlengths'] = allleafpathlengths
all_data['project_pathlength'] = project_pathlength
all_data['somapercent'] = somapercent
all_data['projdata'] = projdata
print "Project Data Collected"
return all_data
#!/usr/bin/env python
import catmaid
src = catmaid.get_source('../../data/skeletons', cache=False)
allsids, PLLnsids, Dnmsids, Onmsids, nucMLFsids, Mauthsids, Anmsids = ([], [],
[], [],
[], [],
[])
for sid in src.skeleton_ids():
neu = src.get_neuron(sid)
if not any([('Blacklist' in anno) for anno in neu.annotations]):
allsids.append(neu.skeleton_id)
# FIXME probably a better way to handle these
# generate lists of mutually exclusive cell descriptions
if any([('PLLn' in anno) for anno in neu.annotations]):
PLLnsids.append(neu.skeleton_id)
elif any([('Dorsal Neuromast' in anno) for anno in neu.annotations]):
Dnmsids.append(neu.skeleton_id)
elif any([('Anterior Neuromast' in anno) for anno in neu.annotations]):
Anmsids.append(neu.skeleton_id)
elif any([('Occipital Neuromast' in anno)
for anno in neu.annotations]):
Onmsids.append(neu.skeleton_id)
elif any([('nucMLF' in anno) for anno in neu.annotations]):
nucMLFsids.append(neu.skeleton_id)
elif any([('Mauthner' in anno) for anno in neu.annotations]):
Mauthsids.append(neu.skeleton_id)
import catmaid
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
import matplotlib.pyplot as plt
import csv
csv_w = csv.writer(open('ax.csv', 'wb'))
csv_2 = csv.writer(open('den.csv', 'wb'))
c = catmaid.connection.Connection('http://catmaid.hms.harvard.edu',
'thomas.lo', 'asdfjkl;', 9)
src = catmaid.get_source(c)
n1 = src.get_neuron(325123)
n2 = src.get_neuron(38321)
g1 = catmaid.algorithms.morphology.resample_edges_2(n1, 400., ['axon'])
g2 = catmaid.algorithms.morphology.resample_edges_2(n2, 400., ['dendrite'])
t_s = catmaid.algorithms.population.synapses.skeleton_overlap_v_verbose_ids(
n1, n2, 1000., 10000., g1, g2)
fig = plt.figure()
ax = fig.gca(projection='3d')
min_r = float(t_s[0][2])
max_r = min_r
sum_r = 0.0
for (_, _, r) in t_s:
min_r = min(min_r, float(r))
max_r = max(max_r, float(r))
sum_r += float(r)
print("With Resampling: SUM:{} MIN:{} MAX:{}".format(sum_r, min_r, max_r))
for (a, b, r) in t_s:
[x1, y1, z1] = [n1.nodes[a][i] for i in ['x', 'y', 'z']]
myFile.write('<html>')
myFile.close()
def fetch_and_check(source):
'''fetch the skeletons to the computer, and check each skeleton'''
sids = fetch_skeletons_pc(source)
problemSkels = check_skeleton_list(sids, source)
return problemSkels
def problem_to_skels(problemSkels):
'''return a dict, key is a problem type, value is a list of skel_ids'''
problem_to_skels_dict = {}
for skel in problemSkels.keys():
problems = problemSkels[skel]
for key in problems.keys():
if key in problem_to_skels_dict.keys():
problem_to_skels_dict[key].append(skel)
else:
problem_to_skels_dict[key] = []
problem_to_skels_dict[key].append(skel)
return problem_to_skels_dict
if __name__ == '__main__':
c = connect_to_catmaid()
src = catmaid.get_source(c, opts.cache)
problem_viewer(src)
import catmaid
#from mpl_toolkits.mplot3d import Axes3D
import csv
import numpy as np
c_a = csv.writer(open('ax.csv','wb'))
c_d = csv.writer(open('den.csv','wb'))
c=catmaid.connection.Connection('http://catmaid.hms.harvard.edu','thomas.lo','asdfjkl;',9)
src = catmaid.get_source(c)
n1 = src.get_neuron(325123)
n2 = src.get_neuron(38321)
g1 = catmaid.algorithms.morphology.resample_edges_2(n1,400.,['axon'])
g2 = catmaid.algorithms.morphology.resample_edges_2(n2,400.,['dendrite'])
t_s = catmaid.algorithms.population.synapses.skeleton_overlap_v_verbose_ids(n1,n2,1000.,10000.,g1,g2)
min_r = float(t_s[0][2])
max_r = min_r
sum_r = 0.0
for (_,_,r) in t_s:
min_r = min(min_r,float(r))
max_r = max(max_r,float(r))
sum_r += float(r)
print("With Resampling: SUM:{} MIN:{} MAX:{}".format(sum_r,min_r,max_r))
for (u,k,r) in t_s:
xyz = []
import argparse
import catmaid
import catmaid.algorithms.images as IM
import os
import scipy.misc
import urllib2
conn = catmaid.connect()
s = catmaid.get_source(conn)
def gen_images(connection, zrange, center, outdir, imgshape=(1024, 1024)):
if not isinstance(zrange, tuple):
raise Exception("Must pass in a tuple for the image range!")
if not isinstance(center, tuple):
raise Exception("Must pass in a tuple for the center position")
broken_slices = [
int(a) for a in conn.stack_info()[6][u'broken_slices'].keys()
]
for z in range(zrange[0], zrange[1]):
if z not in broken_slices:
print "Outputting image for z: {}".format(z)
fn = '{}_sub.png'.format(str(int(z)).zfill(5))
try:
image, no_overlay = IM.img_from_catmaid(connection,
center[0],
center[1],
int(z),
imgshape=imgshape,
stack_id=6,
tiletype=4,
if opts.name is None:
opts.name = os.path.splitext(opts.input)[1]
bn = None
else: # assuming opts.input is a list of skeleton ids all 1 group
logging.debug("Loading input from string: %s", opts.input)
groups = [opts.input.split(',')]
bn = None
logging.debug("Found %s groups", len(groups))
all_sids = []
for g in groups:
all_sids += g
# prepare source
logging.debug("Getting source: %s", opts.source)
s = catmaid.get_source(opts.source)
logging.debug("Preparing source: %s", s)
# TODO figure out a way to check the source to see if conversion is needed
tree_dir, conn_fn, fails = catmaid.rendering.render.prepare_source(s, all_sids)
#tree_dir = opts.source + '/trees'
#conn_fn = opts.source + '/trees/conns.p'
#fails = None
logging.debug("Trees are in %s, conn file is %s", tree_dir, conn_fn)
if opts.conns is not None:
# TODO figure out what to do here
assert opts.conns == conn_fn
# make general kwargs
kwargs = {
'conns_fn': conn_fn,
'attrs_fn': full_path(opts.attrs),
'--threads',
type=int,
required=False,
help="The number of threads to use for smoothing")
opts = parser.parse_args()
if opts.source:
indir = opts.source
else:
indir = None
if opts.dest:
outdir = opts.dest
else:
print "Outdir not passed through. Reverting to default"
outdir = outdir
print "Connecting to source"
s = catmaid.get_source(indir)
if outdir[-1] != '/':
outdir += '/'
if opts.threads:
cpu_count = mp.cpu_count()
if opts.threads > cpu_count:
cpu_count = int(cpu_count / 2)
else:
cpu_count = opts.threads
else:
cpu_count = int(mp.cpu_count() / 2)
if cpu_count < 0:
cpu_count = 1
print "Creating pool operator with {} CPUS".format(cpu_count)
pool = mp.Pool(processes=cpu_count)
print "Creating Smoothing Object"
import catmaid
import scipy.io
import numpy
from httplib import BadStatusLine
# First setup a new Source
src = catmaid.get_source(cache=False)
print 'Pulling Adj and Skeletons from source'
adj, skels = src.get_graph()
scipy.io.savemat('adjacency.mat', mdict={'adjacency': adj})
scipy.io.savemat('skeletons.mat', mdict={'skeletons': skels})
print 'Pulling ApicalList and EMidOriRGBSpeed list from file'
fn_a = 'ApSkList'
fn_e = 'EMidOriRGBneuronIDSFTFspeed'
ApicalList = scipy.io.loadmat(fn_a+'.mat')[fn_a].astype(int)
EMidOriSpd = scipy.io.loadmat(fn_e+'.mat')[fn_e]
c = src._skel_source
somalist = []
COMList = []
print 'Creating SomaList and COMList'
for x in skels:
st = str(x)
print st
try:
urltemp = 'http://catmaid.hms.harvard.edu/9/skeleton/' +\
st + '/get-root'
jsonA = c.fetchJSON(urltemp)
rid = jsonA['root_id']
sep = os.sep
base_dir = "Z:" + sep + "Data" + sep + "simulation" + sep + "tracing"
skeleton_dir = os.path.join(base_dir, 'skeletons')
# Make our base directory
if not os.path.exists(base_dir):
os.makedirs(base_dir)
# Make a directory for our skeletons, nested one deep in our base directory.
# We'll be saving individual skeletons as JSON files here.
if not os.path.exists(skeleton_dir):
os.mkdir(skeleton_dir)
# Get the connection to our Postgre database
thesource = catmaid.get_source()
# Save out all the annotations for our skeletons
annotations_dict = thesource._skel_source.fetchJSON(
'http://catmaid2.hms.harvard.edu/6/neuron/table/query-by-annotations')
with open(os.path.join(base_dir, 'annotations.json'), 'w') as f:
json.dump(annotations_dict, f)
# Pull connectors for all neurons in the project and save them to a json file called connectors
connectors = catmaid.algorithms.population.network.find_conns(
thesource.all_neurons_iter())
with open(os.path.join(base_dir, 'connectors.json'), 'w') as f:
json.dump(connectors, f)
# For every skeleton, get a dictionary representation, and save it to a JSON file
# in our skeletons directory
