def test_features(): global test_file morph = swc.read_swc(test_file) features = MorphologyFeatures(morph) #json.write("out.json", features.apical_dendrite) # check apical features table = features.apical_dendrite errs = 0 for name in names: errs += compare_value(table, name) # check axon features table = features.axon errs = 0 for name in names_axon: errs += compare_value(table, name) num_tests = len(names) # test segments tests, err = check_segments(morph) num_tests += tests errs += err if errs > 0: raise Exception("Failed %d of %d tests" % (errs, num_tests)) print("encountered %d errors in %d tests" % (errs, num_tests))
def run_morphology_summary(pia_transform, relative_soma_depth, soma_depth, swc_file, thumbnail_file, cortex_thumbnail_file, normal_depth_thumbnail_file, high_resolution_thumbnail_file): morphology = swc.read_swc(swc_file) morphology_summary = ms.MorphologySummary(morphology, soma_depth, relative_soma_depth) draw_cortex_thumbnail(morphology_summary, cortex_thumbnail_file, pia_transform) draw_thumbnail(morphology_summary, thumbnail_file, pia_transform, 1, 0, scalebar=True) draw_thumbnail(morphology_summary, high_resolution_thumbnail_file, pia_transform, 5, 0, scalebar=False) draw_normal_depth_thumbnail(morphology_summary, normal_depth_thumbnail_file, pia_transform)
def calculate_transform(db_conn, specimen_id): global cursor cursor = db_conn jin = fetch_json(specimen_id) soma = jin["Soma"]["path"] pia = jin["Pia"]["path"] wm = jin["White Matter"]["path"] res = jin["Pia"]["resolution"] tr_rot, depth = get_pia_wm_rotation_transform(wm_coords=wm, pia_coords=pia, soma_coords=soma, resolution=res) morph = swc.read_swc(jin["swc_file"]) morph.apply_affine(tr_rot) root = morph.soma_root() tr_rot[9] = -root.x tr_rot[10] = -root.y - depth tr_rot[11] = -root.z return tr_rot
def main(jin): # per IT-14567, blockface analysis is no longer required ######################################################################### ## analyze blockface image #try: # soma = jin["blockface"]["Soma"]["path"] # pia = jin["blockface"]["Pia"]["path"] # wm = jin["blockface"]["White Matter"]["path"] # res = float(jin["blockface"]["Pia"]["resolution"]) #except: # print("** Error -- missing requisite blockface field(s) in input json") # raise # ## get soma position using weighted average of vertices #try: # sx, sy = convert_coords_str(soma) # soma_x, soma_y = calculate_centroid(sx, sy) #except: # print("** Error -- unable to calculate soma information (blockface)") # raise # ## calculate shortest path #try: # px, py, wx, wy = calculate_shortest(soma_x, soma_y, pia, wm) ## calculate theta and affine # theta = vector_angle((0, 1), np.asarray([px,py]) - np.asarray([wx,wy])) ## calculate soma depth and cortical thickness # depth = res * euclidean((soma_x, soma_y), (px, py)) # blk_thickness = res * euclidean((wx, wy), (px, py)) #except: # print("** Error calculating shortest path (blockface)") # raise # #blockface = {} #blockface["pia_intersect"] = [ px, py ] #blockface["wm_intersect"] = [ wx, wy ] #blockface["soma_center"] = [ soma_x, soma_y ] #blockface["soma_depth_um"] = depth #try: # blockface["soma_depth_relative"] = depth / blk_thickness #except: # blockface["soma_depth_relative"] = -1.0 # NaN is not friendly to ruby #blockface["cort_thickness_um"] = blk_thickness #blockface["theta"] = theta ######################################################################## # analyze primary (20x) image try: soma = jin["primary"]["Soma"]["path"] pia = jin["primary"]["Pia"]["path"] wm = jin["primary"]["White Matter"]["path"] res = float(jin["primary"]["Pia"]["resolution"]) except: print( "** Error -- missing requisite primary (20x) field(s) in input json" ) raise # get soma position using weighted average of vertices try: sx, sy = convert_coords_str(soma) soma_x, soma_y = calculate_centroid(sx, sy) except: print("** Error -- unable to calculate soma information (primary)") raise try: # calculate shortest path px, py, wx, wy = calculate_shortest(soma_x, soma_y, pia, wm) # calculate theta and affine theta = vector_angle((0, 1), np.asarray([px, py]) - np.asarray([wx, wy])) tr_rot = construct_affine(theta) inv_tr_rot = construct_affine(-theta) # calculate soma depth and cortical thickness depth = res * euclidean((soma_x, soma_y), (px, py)) raw_thickness = res * euclidean((wx, wy), (px, py)) except: print("** Error calculating shortest path (primary)") raise primary = {} primary["pia_intersect"] = [px, py] primary["wm_intersect"] = [wx, wy] primary["soma_center"] = [soma_x, soma_y] primary["soma_depth_um"] = depth try: primary["soma_depth_relative"] = depth / raw_thickness except: primary["soma_depth_relative"] = -1.0 # NaN is not friendly to ruby primary["cort_thickness_um"] = raw_thickness primary["theta"] = theta try: scale = raw_thickness / blk_thickness except: scale = -1.0 # NaN is not ruby-friendly soma_coords_avail = False if "swc_file" in jin: # if SWC file available, extract soma position from it try: nrn = swc.read_swc(jin["swc_file"]) root = nrn.soma_root() soma_x = root.x soma_y = root.y soma_z = root.z soma_coords_avail = True except: # treat this as a fatal error -- if SWC was specified then # it should be used print("**** Error reading SWC file '%s'" % jin["swc_file"]) raise if not soma_coords_avail: # hope that the 63x data is available. As of May 2017, this seems # to no longer be supplied to the module, but just in case... # IT-14567 continue if 63x data not available try: info = jin["soma_63x"] sx, sy = convert_coords_str(info["path_63x"]) soma_x, soma_y = calculate_centroid(sx, sy) soma_x *= float(info["resolution"]) soma_y *= float(info["resolution"]) soma_z = float(info["idx"]) * float(info["thickness"]) soma_coords_avail = True except: print("** Error reading soma 63x info from input json") print("** Translation component of affine matrix is invalid **") if not soma_coords_avail: raise Exception( "** Error: Unable to construct translation component of affine") try: # apply affine rotation to soma position translate_x = soma_x * tr_rot[0] + soma_y * tr_rot[ 1] + soma_z * tr_rot[2] translate_y = soma_x * tr_rot[3] + soma_y * tr_rot[ 4] + soma_z * tr_rot[5] translate_z = soma_x * tr_rot[6] + soma_y * tr_rot[ 7] + soma_z * tr_rot[8] # apply translation vector to transform tr_rot[9] = -translate_x tr_rot[10] = -translate_y - depth tr_rot[11] = -translate_z except: print("** Error calculating affine tranform (math fault?)") raise soma_x = -translate_x soma_y = -translate_y - depth soma_z = -translate_z # apply affine rotation to soma position translate_x = soma_x * inv_tr_rot[0] + soma_y * inv_tr_rot[ 1] + soma_z * inv_tr_rot[2] translate_y = soma_x * inv_tr_rot[3] + soma_y * inv_tr_rot[ 4] + soma_z * inv_tr_rot[5] translate_z = soma_x * inv_tr_rot[6] + soma_y * inv_tr_rot[ 7] + soma_z * inv_tr_rot[8] inv_tr_rot[9] = -translate_x inv_tr_rot[10] = -translate_y inv_tr_rot[11] = -translate_z try: # upright transform. based on rotation in 20x image upright = {} for i in range(12): upright["tvr_%02d" % i] = tr_rot[i] upright["trv_%02d" % i] = inv_tr_rot[i] jout = {} jout["primary"] = primary #jout["blockface"] = blockface # per IT-14567, disable blockface jout["upright"] = upright alignment = {} alignment["scale"] = scale alignment["rotate_x"] = theta alignment["rotate_y"] = theta alignment["rotate_z"] = 0.0 alignment["scale_x"] = 1.0 alignment["scale_y"] = 1.0 alignment["scale_z"] = 1.0 alignment["skew_x"] = 0.0 alignment["skew_y"] = 0.0 alignment["skew_z"] = 0.0 jout["alignment"] = alignment except: print("** Internal error **") raise return jout
def main(jin): try: swc_file = jin["swc_file"] xform = jin["pia_transform"] depth = jin["relative_soma_depth"] except: print("** Unable to find requisite fields in input json") raise #################################################################### # calculate features try: nrn = swc.read_swc(swc_file) except: print("** Error reading swc file") raise try: aff = [] for i in range(12): aff.append(xform["tvr_%02d" % i]) nrn.apply_affine(aff) except: print("** Error applying affine transform") raise #try: # # save a copy of affine-corrected file # tmp_swc_file = swc_file[:-4] + "_pia.swc" # nrn.write(tmp_swc_file) #except: # # treat this as a soft error and print a warning # print("Note: unable to write copy of affine corrected pia file") try: features = feature_extractor.MorphologyFeatures(nrn, depth) data = {} data["axon"] = features.axon data["cloud"] = features.axon_cloud data["dendrite"] = features.dendrite data["basal_dendrite"] = features.basal_dendrite data["apical_dendrite"] = features.apical_dendrite data["all_neurites"] = features.all_neurites except: print("** Error calculating morphology features") raise # make output of new module backwards compatible with previous module md = {} feat = {} feat["number_of_stems"] = data["dendrite"]["num_stems"] feat["max_euclidean_distance"] = data["dendrite"]["max_euclidean_distance"] feat["max_path_distance"] = data["dendrite"]["max_path_distance"] feat["overall_depth"] = data["dendrite"]["depth"] feat["total_volume"] = data["dendrite"]["total_volume"] feat["average_parent_daughter_ratio"] = data["dendrite"][ "mean_parent_daughter_ratio"] feat["average_diameter"] = data["dendrite"]["average_diameter"] feat["total_length"] = data["dendrite"]["total_length"] feat["nodes_over_branches"] = data["dendrite"]["neurites_over_branches"] feat["overall_width"] = data["dendrite"]["width"] feat["number_of_nodes"] = data["dendrite"]["num_nodes"] feat["average_bifurcation_angle_local"] = data["dendrite"][ "bifurcation_angle_local"] feat["number_of_bifurcations"] = data["dendrite"]["num_bifurcations"] feat["average_fragmentation"] = data["dendrite"]["mean_fragmentation"] feat["number_of_tips"] = data["dendrite"]["num_tips"] feat["average_contraction"] = data["dendrite"]["contraction"] feat["average_bifuraction_angle_remote"] = data["dendrite"][ "bifurcation_angle_remote"] feat["number_of_branches"] = data["dendrite"]["num_branches"] feat["total_surface"] = data["dendrite"]["total_surface"] feat["max_branch_order"] = data["dendrite"]["max_branch_order"] feat["soma_surface"] = data["dendrite"]["soma_surface"] feat["overall_height"] = data["dendrite"]["height"] md["features"] = feat data["morphology_data"] = md return data
record["path"] = result[0][3] return record # make upright versions of each morphology for spec_id in spec_ids: dname = "%d" % spec_id if not os.path.isdir(dname): os.makedirs(dname) fname = "%s/%d.swc" % (dname, spec_id) if not os.path.isfile(fname): record = fetch_specimen_record(spec_id) # calculate upright transform aff = prep_upright.calculate_transform(cursor, spec_id) # open SWC file and apply upright transform nrn = swc.read_swc(record["path"] + record["filename"]) nrn.apply_affine(aff) # save morphology nrn.save("%d.swc" % spec_id) ######################################################################## # make thumnails for each image frame and store these in a directory # with the same name as the specimen ID # frame size in pixels width = 800 height = 800 # movie is on a black background, while the soma is normally black # change soma color so it is visible mc = morphvis.MorphologyColors()
def main(jin): global resolution, LINE_WIDTH, DOWNSAMPLE_STEPS jout = {} spec_id = jin["specimen_id"] #################################################################### if "line_width" in jin: LINE_WIDTH = int(jin["line_width"]) # according to Staci, accuracy of 20x trace is approximately to the # level of a cell soma (8-10um), which is approx 22 pixels # downsampling by 2 won't significantly alter the accuracy of # the annotations and is well within the stated margin of error. this # lends itself to more efficient processing, and better thumbnails if "downsample_steps" in jin: DOWNSAMPLE_STEPS = int(jin["downsample_steps"]) ############################################ # derived constants RADS = [] RADS.append(29) RADS.append(15) RADS.append(7) RADS.append(3) DOWNSAMPLE = 1 for i in range(DOWNSAMPLE_STEPS): DOWNSAMPLE *= 2 GAUS_RAD = RADS[DOWNSAMPLE_STEPS] #################################################################### # calculate soma position and store in jin structure soma_res = jin["soma"]["path"] soma_path = soma_res[0].split(',') soma_x = np.array(soma_path[0::2], dtype=float) soma_y = np.array(soma_path[1::2], dtype=float) soma_path = [] for i in range(len(soma_x)): soma_path.append([soma_x[i], soma_y[i]]) soma_path = np.array(soma_path, np.int32) soma_pos = calculate_centroid(soma_x, soma_y) jin["soma"]["position"] = soma_pos resolution = jin["resolution"] swc_name = jin["storage_directory"] + jin["swc_file"] ########################### # before doing the heavy work, load external objects to make sure they're # available # read morphology morph = swc.read_swc(swc_name) # read 20x fname_20x = jin["20x"]["img_path"] image_20x = jpeg_twok.read(fname_20x, reduction_factor=DOWNSAMPLE_STEPS) abs_width = image_20x.shape[1] abs_height = image_20x.shape[0] print("20x image size %dx%d at pyramid level %d" % (abs_width, abs_height, DOWNSAMPLE_STEPS)) # get soma position in 20x pixel space, at present downsample level # resolution converts soma coords in microns to pixels # (resolution is microns / pixel) resolution *= DOWNSAMPLE # divide pixels by X means mult res by same print("Image resolution (microns/pixel): %f" % resolution) dx = jin["soma"]["position"][0] / DOWNSAMPLE - morph.soma_root( ).x / resolution dy = jin["soma"]["position"][1] / DOWNSAMPLE - morph.soma_root( ).y / resolution dx = int(dx) dy = int(dy) ############################## # no point in processing the entire image, as only a small part # is relevant # select min/max values for x,y of all polygons. restrict analysis # to there min_x = 1e10 min_y = 1e10 max_x = 0 max_y = 0 layers = jin["layers"] for layer in layers: path_array = np.array(layer["path"].split(',')) x = np.array(path_array[0::2], dtype=float) y = np.array(path_array[1::2], dtype=float) min_x = min(min_x, x.min()) min_y = min(min_y, y.min()) max_x = max(max_x, x.max()) max_y = max(max_y, y.max()) min_x /= DOWNSAMPLE min_y /= DOWNSAMPLE max_x /= DOWNSAMPLE max_y /= DOWNSAMPLE # add a border around polygons to provide context BORDER = 200 BORDER /= DOWNSAMPLE TOP = min_y - BORDER LEFT = min_x - BORDER RIGHT = max_x + BORDER BOTTOM = max_y + BORDER # make sure border doesn't extend beyond image limits TOP = max(TOP, 0) LEFT = max(LEFT, 0) RIGHT = min(RIGHT, abs_width - 1) BOTTOM = min(BOTTOM, abs_height - 1) WIDTH = int(RIGHT - LEFT) HEIGHT = int(BOTTOM - TOP) # adjust soma location for top and left of visible image area dx -= LEFT dy -= TOP #print "inset soma position", dx, dy # make frame for each polygon and blur. blur radious should be # approx the size of largest gap or overlap between polygons. this # is for estimating which polygon each point is a best fit in layers = jin["layers"] for layer in layers: path_array = np.array(layer["path"].split(',')) x = np.array(path_array[0::2], dtype=float) x /= DOWNSAMPLE x -= LEFT y = np.array(path_array[1::2], dtype=float) y /= DOWNSAMPLE y -= TOP #print layer["label"] #print x.min(), y.min() #print x.max(), y.max() xy = [] for i in range(len(x)): xy.append([x[i], y[i]]) raw_frame = np.zeros((HEIGHT, WIDTH)) path = np.array(xy) cv2.fillPoly(raw_frame, np.int32([path]), 255) frame = cv2.blur(raw_frame, (GAUS_RAD, GAUS_RAD)) layer["frame"] = frame # blurred polygon layer["raw_frame"] = raw_frame # raw polygon # collapse all polys into single array, with value at each position # corresponding to the index of the polygon that the pixel falls # into, or -1 if there's no match master = np.zeros((HEIGHT, WIDTH, 3)) master_idx = np.zeros((HEIGHT, WIDTH), dtype=int) master_idx -= 1 for y in range(HEIGHT): for x in range(WIDTH): peak = 0 idx = -1 for i in range(len(layers)): frame = layers[i]["frame"] val = frame[y][x] if val > peak: peak = val idx = i if idx >= 0: master[y][x] = color_by_index(idx) master_idx[y][x] = idx ################################################# # draw standard morphology on colored layers draw_morphology(morph, master, int(dx), int(dy)) outfile = "layer_%d.png" % spec_id print("saving " + outfile) cv2.imwrite(outfile, master) jout["morph_layers"] = outfile ################################################# # draw standard morphology on 20x image img = image_20x[TOP:BOTTOM, LEFT:RIGHT] print(img.shape) draw_morphology(morph, img, int(dx), int(dy)) outfile = "blockface_%d.png" % spec_id print("saving " + outfile) cv2.imwrite(outfile, img) jout["morph_20x"] = outfile ################################################# # associate SWC nodes with morphology layers jout["reconstruction_id"] = jin["reconstruction_id"] reconstruction = {} errs = 0 for n in morph.node_list: x = dx + int(n.x / resolution) y = dy + int(n.y / resolution) desc = n.to_dict() idx = -1 try: idx = master_idx[y][x] except: errs += 1 if idx >= 0: desc["label"] = layers[idx]["label"] n.layer_num = idx else: desc["label"] = "unknown" n.layer_num = -1 reconstruction[n.n] = desc if errs > 0: raise Exception("Unable to map %d nodes to a cortical layer" % errs) jout["reconstruction"] = reconstruction ################################################# # draw layer & morphology SVG outfile = "outline_%d.svg" % spec_id print("saving " + outfile) jout["outline_svg"] = outfile write_svg(outfile, jin, morph) ################################################# # draw layer-colored morphology on 20x image img = image_20x[TOP:BOTTOM, LEFT:RIGHT] draw_morphology(morph, img, int(dx), int(dy), True) outfile = "layered_blockface_%d.png" % spec_id print("saving " + outfile) cv2.imwrite(outfile, img) jout["colored_morph_20x"] = outfile ################################################# # draw layer-colored morphology on empty polygons img = np.zeros((HEIGHT, WIDTH, 3)) layers = jin["layers"] for layer in layers: path_array = np.array(layer["path"].split(',')) x = np.array(path_array[0::2], dtype=float) x /= DOWNSAMPLE x -= LEFT y = np.array(path_array[1::2], dtype=float) y /= DOWNSAMPLE y -= TOP for i in range(1, len(x)): cv2.line(img, (int(x[i - 1]), int(y[i - 1])), (int(x[i]), int(y[i])), (255, 255, 255), 1) cv2.line(img, (int(x[-1]), int(y[-1])), (int(x[0]), int(y[0])), (255, 255, 255), 1) draw_morphology(morph, img, int(dx), int(dy), True) outfile = "outline_%d.png" % spec_id print("saving " + outfile) cv2.imwrite(outfile, img) jout["colored_morph_poly"] = outfile return jout
return results # extract feature data # global dictionary to store features. one entry per specimen_id morph_data = {} for k, record in records.iteritems(): # get SWC spec_id = record["spec_id"] if spec_id in ids_to_ignore: print("-- Cell %d is in the ignore list so skipping it" % spec_id) swc_file = record["path"] + record["filename"] print("Processing '%s'" % swc_file) try: nrn = swc.read_swc(swc_file) except Exception, e: #print e print("") print("**** Error: problem encountered open specified file ****") print("Specimen id: %d" % spec_id) print("Specimen name: " + record["spec_name"]) print("Specimen path: " + record["path"]) print("Specimen file: " + record["filename"]) print("") print(traceback.print_exc()) print("-----------------------------------------------------------") continue # process features try: # apply affine transform, if appropriate