def test_features():
global test_file
morph = swc.read_swc(test_file)
features = MorphologyFeatures(morph)
#json.write("out.json", features.apical_dendrite)
# check apical features
table = features.apical_dendrite
errs = 0
for name in names:
errs += compare_value(table, name)
# check axon features
table = features.axon
errs = 0
for name in names_axon:
errs += compare_value(table, name)
num_tests = len(names)
# test segments
tests, err = check_segments(morph)
num_tests += tests
errs += err
if errs > 0:
raise Exception("Failed %d of %d tests" % (errs, num_tests))
print("encountered %d errors in %d tests" % (errs, num_tests))
def run_morphology_summary(pia_transform, relative_soma_depth, soma_depth,
swc_file, thumbnail_file, cortex_thumbnail_file,
normal_depth_thumbnail_file,
high_resolution_thumbnail_file):
morphology = swc.read_swc(swc_file)
morphology_summary = ms.MorphologySummary(morphology, soma_depth,
relative_soma_depth)
draw_cortex_thumbnail(morphology_summary, cortex_thumbnail_file,
pia_transform)
draw_thumbnail(morphology_summary,
thumbnail_file,
pia_transform,
1,
0,
scalebar=True)
draw_thumbnail(morphology_summary,
high_resolution_thumbnail_file,
pia_transform,
5,
0,
scalebar=False)
draw_normal_depth_thumbnail(morphology_summary,
normal_depth_thumbnail_file, pia_transform)
def calculate_transform(db_conn, specimen_id):
global cursor
cursor = db_conn
jin = fetch_json(specimen_id)
soma = jin["Soma"]["path"]
pia = jin["Pia"]["path"]
wm = jin["White Matter"]["path"]
res = jin["Pia"]["resolution"]
tr_rot, depth = get_pia_wm_rotation_transform(wm_coords=wm,
pia_coords=pia,
soma_coords=soma,
resolution=res)
morph = swc.read_swc(jin["swc_file"])
morph.apply_affine(tr_rot)
root = morph.soma_root()
tr_rot[9] = -root.x
tr_rot[10] = -root.y - depth
tr_rot[11] = -root.z
return tr_rot
def main(jin):
# per IT-14567, blockface analysis is no longer required
#########################################################################
## analyze blockface image
#try:
# soma = jin["blockface"]["Soma"]["path"]
# pia = jin["blockface"]["Pia"]["path"]
# wm = jin["blockface"]["White Matter"]["path"]
# res = float(jin["blockface"]["Pia"]["resolution"])
#except:
# print("** Error -- missing requisite blockface field(s) in input json")
# raise
#
## get soma position using weighted average of vertices
#try:
# sx, sy = convert_coords_str(soma)
# soma_x, soma_y = calculate_centroid(sx, sy)
#except:
# print("** Error -- unable to calculate soma information (blockface)")
# raise
#
## calculate shortest path
#try:
# px, py, wx, wy = calculate_shortest(soma_x, soma_y, pia, wm)
## calculate theta and affine
# theta = vector_angle((0, 1), np.asarray([px,py]) - np.asarray([wx,wy]))
## calculate soma depth and cortical thickness
# depth = res * euclidean((soma_x, soma_y), (px, py))
# blk_thickness = res * euclidean((wx, wy), (px, py))
#except:
# print("** Error calculating shortest path (blockface)")
# raise
#
#blockface = {}
#blockface["pia_intersect"] = [ px, py ]
#blockface["wm_intersect"] = [ wx, wy ]
#blockface["soma_center"] = [ soma_x, soma_y ]
#blockface["soma_depth_um"] = depth
#try:
# blockface["soma_depth_relative"] = depth / blk_thickness
#except:
# blockface["soma_depth_relative"] = -1.0 # NaN is not friendly to ruby
#blockface["cort_thickness_um"] = blk_thickness
#blockface["theta"] = theta
########################################################################
# analyze primary (20x) image
try:
soma = jin["primary"]["Soma"]["path"]
pia = jin["primary"]["Pia"]["path"]
wm = jin["primary"]["White Matter"]["path"]
res = float(jin["primary"]["Pia"]["resolution"])
except:
print(
"** Error -- missing requisite primary (20x) field(s) in input json"
)
raise
# get soma position using weighted average of vertices
try:
sx, sy = convert_coords_str(soma)
soma_x, soma_y = calculate_centroid(sx, sy)
except:
print("** Error -- unable to calculate soma information (primary)")
raise
try:
# calculate shortest path
px, py, wx, wy = calculate_shortest(soma_x, soma_y, pia, wm)
# calculate theta and affine
theta = vector_angle((0, 1),
np.asarray([px, py]) - np.asarray([wx, wy]))
tr_rot = construct_affine(theta)
inv_tr_rot = construct_affine(-theta)
# calculate soma depth and cortical thickness
depth = res * euclidean((soma_x, soma_y), (px, py))
raw_thickness = res * euclidean((wx, wy), (px, py))
except:
print("** Error calculating shortest path (primary)")
raise
primary = {}
primary["pia_intersect"] = [px, py]
primary["wm_intersect"] = [wx, wy]
primary["soma_center"] = [soma_x, soma_y]
primary["soma_depth_um"] = depth
try:
primary["soma_depth_relative"] = depth / raw_thickness
except:
primary["soma_depth_relative"] = -1.0 # NaN is not friendly to ruby
primary["cort_thickness_um"] = raw_thickness
primary["theta"] = theta
try:
scale = raw_thickness / blk_thickness
except:
scale = -1.0 # NaN is not ruby-friendly
soma_coords_avail = False
if "swc_file" in jin:
# if SWC file available, extract soma position from it
try:
nrn = swc.read_swc(jin["swc_file"])
root = nrn.soma_root()
soma_x = root.x
soma_y = root.y
soma_z = root.z
soma_coords_avail = True
except:
# treat this as a fatal error -- if SWC was specified then
# it should be used
print("**** Error reading SWC file '%s'" % jin["swc_file"])
raise
if not soma_coords_avail:
# hope that the 63x data is available. As of May 2017, this seems
# to no longer be supplied to the module, but just in case...
# IT-14567 continue if 63x data not available
try:
info = jin["soma_63x"]
sx, sy = convert_coords_str(info["path_63x"])
soma_x, soma_y = calculate_centroid(sx, sy)
soma_x *= float(info["resolution"])
soma_y *= float(info["resolution"])
soma_z = float(info["idx"]) * float(info["thickness"])
soma_coords_avail = True
except:
print("** Error reading soma 63x info from input json")
print("** Translation component of affine matrix is invalid **")
if not soma_coords_avail:
raise Exception(
"** Error: Unable to construct translation component of affine")
try:
# apply affine rotation to soma position
translate_x = soma_x * tr_rot[0] + soma_y * tr_rot[
1] + soma_z * tr_rot[2]
translate_y = soma_x * tr_rot[3] + soma_y * tr_rot[
4] + soma_z * tr_rot[5]
translate_z = soma_x * tr_rot[6] + soma_y * tr_rot[
7] + soma_z * tr_rot[8]
# apply translation vector to transform
tr_rot[9] = -translate_x
tr_rot[10] = -translate_y - depth
tr_rot[11] = -translate_z
except:
print("** Error calculating affine tranform (math fault?)")
raise
soma_x = -translate_x
soma_y = -translate_y - depth
soma_z = -translate_z
# apply affine rotation to soma position
translate_x = soma_x * inv_tr_rot[0] + soma_y * inv_tr_rot[
1] + soma_z * inv_tr_rot[2]
translate_y = soma_x * inv_tr_rot[3] + soma_y * inv_tr_rot[
4] + soma_z * inv_tr_rot[5]
translate_z = soma_x * inv_tr_rot[6] + soma_y * inv_tr_rot[
7] + soma_z * inv_tr_rot[8]
inv_tr_rot[9] = -translate_x
inv_tr_rot[10] = -translate_y
inv_tr_rot[11] = -translate_z
try:
# upright transform. based on rotation in 20x image
upright = {}
for i in range(12):
upright["tvr_%02d" % i] = tr_rot[i]
upright["trv_%02d" % i] = inv_tr_rot[i]
jout = {}
jout["primary"] = primary
#jout["blockface"] = blockface # per IT-14567, disable blockface
jout["upright"] = upright
alignment = {}
alignment["scale"] = scale
alignment["rotate_x"] = theta
alignment["rotate_y"] = theta
alignment["rotate_z"] = 0.0
alignment["scale_x"] = 1.0
alignment["scale_y"] = 1.0
alignment["scale_z"] = 1.0
alignment["skew_x"] = 0.0
alignment["skew_y"] = 0.0
alignment["skew_z"] = 0.0
jout["alignment"] = alignment
except:
print("** Internal error **")
raise
return jout
def main(jin):
try:
swc_file = jin["swc_file"]
xform = jin["pia_transform"]
depth = jin["relative_soma_depth"]
except:
print("** Unable to find requisite fields in input json")
raise
####################################################################
# calculate features
try:
nrn = swc.read_swc(swc_file)
except:
print("** Error reading swc file")
raise
try:
aff = []
for i in range(12):
aff.append(xform["tvr_%02d" % i])
nrn.apply_affine(aff)
except:
print("** Error applying affine transform")
raise
#try:
# # save a copy of affine-corrected file
# tmp_swc_file = swc_file[:-4] + "_pia.swc"
# nrn.write(tmp_swc_file)
#except:
# # treat this as a soft error and print a warning
# print("Note: unable to write copy of affine corrected pia file")
try:
features = feature_extractor.MorphologyFeatures(nrn, depth)
data = {}
data["axon"] = features.axon
data["cloud"] = features.axon_cloud
data["dendrite"] = features.dendrite
data["basal_dendrite"] = features.basal_dendrite
data["apical_dendrite"] = features.apical_dendrite
data["all_neurites"] = features.all_neurites
except:
print("** Error calculating morphology features")
raise
# make output of new module backwards compatible with previous module
md = {}
feat = {}
feat["number_of_stems"] = data["dendrite"]["num_stems"]
feat["max_euclidean_distance"] = data["dendrite"]["max_euclidean_distance"]
feat["max_path_distance"] = data["dendrite"]["max_path_distance"]
feat["overall_depth"] = data["dendrite"]["depth"]
feat["total_volume"] = data["dendrite"]["total_volume"]
feat["average_parent_daughter_ratio"] = data["dendrite"][
"mean_parent_daughter_ratio"]
feat["average_diameter"] = data["dendrite"]["average_diameter"]
feat["total_length"] = data["dendrite"]["total_length"]
feat["nodes_over_branches"] = data["dendrite"]["neurites_over_branches"]
feat["overall_width"] = data["dendrite"]["width"]
feat["number_of_nodes"] = data["dendrite"]["num_nodes"]
feat["average_bifurcation_angle_local"] = data["dendrite"][
"bifurcation_angle_local"]
feat["number_of_bifurcations"] = data["dendrite"]["num_bifurcations"]
feat["average_fragmentation"] = data["dendrite"]["mean_fragmentation"]
feat["number_of_tips"] = data["dendrite"]["num_tips"]
feat["average_contraction"] = data["dendrite"]["contraction"]
feat["average_bifuraction_angle_remote"] = data["dendrite"][
"bifurcation_angle_remote"]
feat["number_of_branches"] = data["dendrite"]["num_branches"]
feat["total_surface"] = data["dendrite"]["total_surface"]
feat["max_branch_order"] = data["dendrite"]["max_branch_order"]
feat["soma_surface"] = data["dendrite"]["soma_surface"]
feat["overall_height"] = data["dendrite"]["height"]
md["features"] = feat
data["morphology_data"] = md
return data
record["path"] = result[0][3]
return record
# make upright versions of each morphology
for spec_id in spec_ids:
dname = "%d" % spec_id
if not os.path.isdir(dname):
os.makedirs(dname)
fname = "%s/%d.swc" % (dname, spec_id)
if not os.path.isfile(fname):
record = fetch_specimen_record(spec_id)
# calculate upright transform
aff = prep_upright.calculate_transform(cursor, spec_id)
# open SWC file and apply upright transform
nrn = swc.read_swc(record["path"] + record["filename"])
nrn.apply_affine(aff)
# save morphology
nrn.save("%d.swc" % spec_id)
########################################################################
# make thumnails for each image frame and store these in a directory
# with the same name as the specimen ID
# frame size in pixels
width = 800
height = 800
# movie is on a black background, while the soma is normally black
# change soma color so it is visible
mc = morphvis.MorphologyColors()
def main(jin):
global resolution, LINE_WIDTH, DOWNSAMPLE_STEPS
jout = {}
spec_id = jin["specimen_id"]
####################################################################
if "line_width" in jin:
LINE_WIDTH = int(jin["line_width"])
# according to Staci, accuracy of 20x trace is approximately to the
# level of a cell soma (8-10um), which is approx 22 pixels
# downsampling by 2 won't significantly alter the accuracy of
# the annotations and is well within the stated margin of error. this
# lends itself to more efficient processing, and better thumbnails
if "downsample_steps" in jin:
DOWNSAMPLE_STEPS = int(jin["downsample_steps"])
############################################
# derived constants
RADS = []
RADS.append(29)
RADS.append(15)
RADS.append(7)
RADS.append(3)
DOWNSAMPLE = 1
for i in range(DOWNSAMPLE_STEPS):
DOWNSAMPLE *= 2
GAUS_RAD = RADS[DOWNSAMPLE_STEPS]
####################################################################
# calculate soma position and store in jin structure
soma_res = jin["soma"]["path"]
soma_path = soma_res[0].split(',')
soma_x = np.array(soma_path[0::2], dtype=float)
soma_y = np.array(soma_path[1::2], dtype=float)
soma_path = []
for i in range(len(soma_x)):
soma_path.append([soma_x[i], soma_y[i]])
soma_path = np.array(soma_path, np.int32)
soma_pos = calculate_centroid(soma_x, soma_y)
jin["soma"]["position"] = soma_pos
resolution = jin["resolution"]
swc_name = jin["storage_directory"] + jin["swc_file"]
###########################
# before doing the heavy work, load external objects to make sure they're
# available
# read morphology
morph = swc.read_swc(swc_name)
# read 20x
fname_20x = jin["20x"]["img_path"]
image_20x = jpeg_twok.read(fname_20x, reduction_factor=DOWNSAMPLE_STEPS)
abs_width = image_20x.shape[1]
abs_height = image_20x.shape[0]
print("20x image size %dx%d at pyramid level %d" %
(abs_width, abs_height, DOWNSAMPLE_STEPS))
# get soma position in 20x pixel space, at present downsample level
# resolution converts soma coords in microns to pixels
# (resolution is microns / pixel)
resolution *= DOWNSAMPLE # divide pixels by X means mult res by same
print("Image resolution (microns/pixel): %f" % resolution)
dx = jin["soma"]["position"][0] / DOWNSAMPLE - morph.soma_root(
).x / resolution
dy = jin["soma"]["position"][1] / DOWNSAMPLE - morph.soma_root(
).y / resolution
dx = int(dx)
dy = int(dy)
##############################
# no point in processing the entire image, as only a small part
# is relevant
# select min/max values for x,y of all polygons. restrict analysis
# to there
min_x = 1e10
min_y = 1e10
max_x = 0
max_y = 0
layers = jin["layers"]
for layer in layers:
path_array = np.array(layer["path"].split(','))
x = np.array(path_array[0::2], dtype=float)
y = np.array(path_array[1::2], dtype=float)
min_x = min(min_x, x.min())
min_y = min(min_y, y.min())
max_x = max(max_x, x.max())
max_y = max(max_y, y.max())
min_x /= DOWNSAMPLE
min_y /= DOWNSAMPLE
max_x /= DOWNSAMPLE
max_y /= DOWNSAMPLE
# add a border around polygons to provide context
BORDER = 200
BORDER /= DOWNSAMPLE
TOP = min_y - BORDER
LEFT = min_x - BORDER
RIGHT = max_x + BORDER
BOTTOM = max_y + BORDER
# make sure border doesn't extend beyond image limits
TOP = max(TOP, 0)
LEFT = max(LEFT, 0)
RIGHT = min(RIGHT, abs_width - 1)
BOTTOM = min(BOTTOM, abs_height - 1)
WIDTH = int(RIGHT - LEFT)
HEIGHT = int(BOTTOM - TOP)
# adjust soma location for top and left of visible image area
dx -= LEFT
dy -= TOP
#print "inset soma position", dx, dy
# make frame for each polygon and blur. blur radious should be
# approx the size of largest gap or overlap between polygons. this
# is for estimating which polygon each point is a best fit in
layers = jin["layers"]
for layer in layers:
path_array = np.array(layer["path"].split(','))
x = np.array(path_array[0::2], dtype=float)
x /= DOWNSAMPLE
x -= LEFT
y = np.array(path_array[1::2], dtype=float)
y /= DOWNSAMPLE
y -= TOP
#print layer["label"]
#print x.min(), y.min()
#print x.max(), y.max()
xy = []
for i in range(len(x)):
xy.append([x[i], y[i]])
raw_frame = np.zeros((HEIGHT, WIDTH))
path = np.array(xy)
cv2.fillPoly(raw_frame, np.int32([path]), 255)
frame = cv2.blur(raw_frame, (GAUS_RAD, GAUS_RAD))
layer["frame"] = frame # blurred polygon
layer["raw_frame"] = raw_frame # raw polygon
# collapse all polys into single array, with value at each position
# corresponding to the index of the polygon that the pixel falls
# into, or -1 if there's no match
master = np.zeros((HEIGHT, WIDTH, 3))
master_idx = np.zeros((HEIGHT, WIDTH), dtype=int)
master_idx -= 1
for y in range(HEIGHT):
for x in range(WIDTH):
peak = 0
idx = -1
for i in range(len(layers)):
frame = layers[i]["frame"]
val = frame[y][x]
if val > peak:
peak = val
idx = i
if idx >= 0:
master[y][x] = color_by_index(idx)
master_idx[y][x] = idx
#################################################
# draw standard morphology on colored layers
draw_morphology(morph, master, int(dx), int(dy))
outfile = "layer_%d.png" % spec_id
print("saving " + outfile)
cv2.imwrite(outfile, master)
jout["morph_layers"] = outfile
#################################################
# draw standard morphology on 20x image
img = image_20x[TOP:BOTTOM, LEFT:RIGHT]
print(img.shape)
draw_morphology(morph, img, int(dx), int(dy))
outfile = "blockface_%d.png" % spec_id
print("saving " + outfile)
cv2.imwrite(outfile, img)
jout["morph_20x"] = outfile
#################################################
# associate SWC nodes with morphology layers
jout["reconstruction_id"] = jin["reconstruction_id"]
reconstruction = {}
errs = 0
for n in morph.node_list:
x = dx + int(n.x / resolution)
y = dy + int(n.y / resolution)
desc = n.to_dict()
idx = -1
try:
idx = master_idx[y][x]
except:
errs += 1
if idx >= 0:
desc["label"] = layers[idx]["label"]
n.layer_num = idx
else:
desc["label"] = "unknown"
n.layer_num = -1
reconstruction[n.n] = desc
if errs > 0:
raise Exception("Unable to map %d nodes to a cortical layer" % errs)
jout["reconstruction"] = reconstruction
#################################################
# draw layer & morphology SVG
outfile = "outline_%d.svg" % spec_id
print("saving " + outfile)
jout["outline_svg"] = outfile
write_svg(outfile, jin, morph)
#################################################
# draw layer-colored morphology on 20x image
img = image_20x[TOP:BOTTOM, LEFT:RIGHT]
draw_morphology(morph, img, int(dx), int(dy), True)
outfile = "layered_blockface_%d.png" % spec_id
print("saving " + outfile)
cv2.imwrite(outfile, img)
jout["colored_morph_20x"] = outfile
#################################################
# draw layer-colored morphology on empty polygons
img = np.zeros((HEIGHT, WIDTH, 3))
layers = jin["layers"]
for layer in layers:
path_array = np.array(layer["path"].split(','))
x = np.array(path_array[0::2], dtype=float)
x /= DOWNSAMPLE
x -= LEFT
y = np.array(path_array[1::2], dtype=float)
y /= DOWNSAMPLE
y -= TOP
for i in range(1, len(x)):
cv2.line(img, (int(x[i - 1]), int(y[i - 1])),
(int(x[i]), int(y[i])), (255, 255, 255), 1)
cv2.line(img, (int(x[-1]), int(y[-1])), (int(x[0]), int(y[0])),
(255, 255, 255), 1)
draw_morphology(morph, img, int(dx), int(dy), True)
outfile = "outline_%d.png" % spec_id
print("saving " + outfile)
cv2.imwrite(outfile, img)
jout["colored_morph_poly"] = outfile
return jout
return results
# extract feature data
# global dictionary to store features. one entry per specimen_id
morph_data = {}
for k, record in records.iteritems():
# get SWC
spec_id = record["spec_id"]
if spec_id in ids_to_ignore:
print("-- Cell %d is in the ignore list so skipping it" % spec_id)
swc_file = record["path"] + record["filename"]
print("Processing '%s'" % swc_file)
try:
nrn = swc.read_swc(swc_file)
except Exception, e:
#print e
print("")
print("**** Error: problem encountered open specified file ****")
print("Specimen id: %d" % spec_id)
print("Specimen name: " + record["spec_name"])
print("Specimen path: " + record["path"])
print("Specimen file: " + record["filename"])
print("")
print(traceback.print_exc())
print("-----------------------------------------------------------")
continue
# process features
try:
# apply affine transform, if appropriate