def CombineSurfaceVoxels(data):
# start timing statistics
total_time = time.time()
# create new directory for the surfaces
surface_directory = data.SurfacesDirectory()
if not os.path.exists(surface_directory):
os.makedirs(surface_directory, exist_ok=True)
# iterate over all labels
for label in range(1, data.NLabels()):
global_indices = []
# iterate over all blocks and read the points
for iz in range(data.StartZ(), data.EndZ()):
for iy in range(data.StartY(), data.EndY()):
for ix in range(data.StartX(), data.EndX()):
tmp_block_directory = data.TempBlockDirectory(iz, iy, ix)
block_surface_filename = '{}/surfaces/{:016d}.pts'.format(tmp_block_directory, label)
# skip if this file does not exist (this label does not occur in this block)
if not os.path.exists(block_surface_filename): continue
global_block_indices, _ = ReadPtsFile(data, block_surface_filename)
global_indices += global_block_indices[label]
if len(global_indices):
surface_filename = '{}/{:016d}.pts'.format(surface_directory, label)
WritePtsFile(data, surface_filename, { label: global_indices }, input_local_indices = False)
total_time = time.time() - total_time
print ('Combined Surfaces Files: {:0.2f} seconds.'.format(total_time))
def ReadSynpasestoList(data, iz, iy, ix):
# read in synapses for this block
synapse_coordinates = []
fname_synapses = '{}/{:04d}z-{:04d}y-{:04d}x.pts'.format(
data.SynapseDirectory(), iz, iy, ix)
_, synapse_local_indices = ReadPtsFile(data, fname_synapses)
for ID in synapse_local_indices.keys():
for index in synapse_local_indices[ID]:
synapse_coordinates.insert(0, data.LocalIndexToIndices(index))
print("inserted {} synapses".format(len(synapse_coordinates)))
return synapse_coordinates
def EvaluateWidths(data, label):
# get the resolution of this data
resolution = data.Resolution()
# read the width attributes filename
widths_directory = '{}/widths'.format(data.SkeletonOutputDirectory())
width_filename = '{}/{:016d}.pts'.format(widths_directory, label)
# skip over labels not processed
if not os.path.exists(width_filename): return
# read the width attributes
widths, input_label = ReadAttributePtsFile(data, width_filename)
assert (input_label == label)
# get the surface filename
surfaces_filename = '{}/{:016d}.pts'.format(data.SurfacesDirectory(), label)
# read the surfaces, ignore local coordinates
surfaces, _ = ReadPtsFile(data, surfaces_filename)
surface = surfaces[label]
npoints = len(surface)
# convert the surface into a numpy point cloud
np_point_cloud = np.zeros((npoints, 3), dtype=np.float32)
for index, iv in enumerate(surface):
# convert the index into indices
iz, iy, ix = data.GlobalIndexToIndices(iv)
# set the point cloud value according to the resolutions
np_point_cloud[index,:] = (resolution[OR_Z] * iz, resolution[OR_Y] * iy, resolution[OR_X] * ix)
# create empty dictionary for all results
results = {}
# keep track of all errors for this label
results['errors'] = []
results['estimates'] = 0
results['ground_truths'] = 0
# iterate over all skeleton points
for iv in widths.keys():
# get the estimated width at this location
width = widths[iv]
iz, iy, ix = data.GlobalIndexToIndices(iv)
# convert the coordinates into a 2d vector with the resolutions
vec = np.zeros((1, 3), dtype=np.float32)
vec[0,:] = (resolution[OR_Z] * iz, resolution[OR_Y] * iy, resolution[OR_X] * ix)
# get the min distance from this point to the surface (true width)
min_distance = scipy.spatial.distance.cdist(np_point_cloud, vec).min()
results['errors'].append(abs(width - min_distance))
results['estimates'] += width
results['ground_truths'] += min_distance
# skip over vacuous skeletons
if len(results['errors']) < 2: return
# output the errors, estimates, and ground truths to a pickled file
tmp_widths_directory = '{}/results/widths'.format(data.TempDirectory())
if not os.path.exists(tmp_widths_directory):
os.makedirs(tmp_widths_directory, exist_ok=True)
output_filename = '{}/{:016d}.pickle'.format(tmp_widths_directory, label)
PickleData(results, output_filename)
def EvaluateNeuralReconstructionIntegrity(meta_filename):
data = ReadMetaData(meta_filename)
synapses_per_label = {}
# read in all of the synapses from all of the blocks
synapse_directory = data.SynapseDirectory()
for iz in range(data.StartZ(), data.EndZ()):
for iy in range(data.StartY(), data.EndY()):
for ix in range(data.StartX(), data.EndX()):
synapses_filename = '{}/{:04d}z-{:04d}y-{:04d}x.pts'.format(synapse_directory, iz, iy, ix)
# ignore the local coordinates
block_synapses, _ = ReadPtsFile(data, synapses_filename)
# add all synapses for each label in this block to the global synapses
for label in block_synapses.keys():
if not label in synapses_per_label:
synapses_per_label[label] = []
synapses_per_label[label] += block_synapses[label]
# get the output filename
evaluation_directory = data.EvaluationDirectory()
if not os.path.exists(evaluation_directory):
os.makedirs(evaluation_directory, exist_ok=True)
output_filename = '{}/nri-results.txt'.format(evaluation_directory)
fd = open(output_filename, 'w')
# keep track of global statistics
total_true_positives = 0
total_false_positives = 0
total_false_negatives = 0
# for each label, find if synapses correspond to endpoints
for label in range(1, data.NLabels()):
# read the refined skeleton for this synapse
skeleton_directory = '{}/skeletons'.format(data.SkeletonOutputDirectory())
skeleton_filename = '{}/{:016d}.pts'.format(skeleton_directory, label)
# skip over labels not processed
if not os.path.exists(skeleton_filename): continue
# get the synapses only for this one label
synapses = synapses_per_label[label]
# ignore the local coordinates
skeletons, _ = ReadPtsFile(data, skeleton_filename)
skeleton = skeletons[label]
# read in the somata surfaces (points on the surface should not count as endpoints)
somata_directory = '{}/somata_surfaces'.format(data.TempDirectory())
somata_filename = '{}/{:016d}.pts'.format(somata_directory, label)
# path may not exist if soma not found
if os.path.exists(somata_filename):
somata_surfaces, _ = ReadPtsFile(data, somata_filename)
somata_surface = set(somata_surfaces[label])
else:
somata_surface = set()
# get the endpoints in this skeleton for this label
endpoints = FindEndpoints(data, skeleton, somata_surface)
true_positives, false_positives, false_negatives = CalculateNeuralReconstructionIntegrityScore(data, synapses, endpoints)
# if there are no true positives the NRI score is 0
if true_positives == 0:
nri_score = 0
else:
precision = true_positives / float(true_positives + false_positives)
recall = true_positives / float(true_positives + false_negatives)
nri_score = 2 * (precision * recall) / (precision + recall)
print ('Label: {}'.format(label))
print (' True Positives: {:10d}'.format(true_positives))
print (' False Positives: {:10d}'.format(false_positives))
print (' False Negatives: {:10d}'.format(false_negatives))
print (' NRI Score: {:0.4f}'.format(nri_score))
fd.write ('Label: {}\n'.format(label))
fd.write (' True Positives: {:10d}\n'.format(true_positives))
fd.write (' False Positives: {:10d}\n'.format(false_positives))
fd.write (' False Negatives: {:10d}\n'.format(false_negatives))
fd.write (' NRI Score: {:0.4f}\n'.format(nri_score))
# update the global stats
total_true_positives += true_positives
total_false_positives += false_positives
total_false_negatives += false_negatives
precision = total_true_positives / float(total_true_positives + total_false_positives)
recall = total_true_positives / float(total_true_positives + total_false_negatives)
nri_score = 2 * (precision * recall) / (precision + recall)
print ('Total Volume'.format(label))
print (' True Positives: {:10d}'.format(total_true_positives))
print (' False Positives: {:10d}'.format(total_false_positives))
print (' False Negatives: {:10d}'.format(total_false_negatives))
print (' NRI Score: {:0.4f}'.format(nri_score))
fd.write ('Total Volume\n'.format(label))
fd.write (' True Positives: {:10d}\n'.format(total_true_positives))
fd.write (' False Positives: {:10d}\n'.format(total_false_positives))
fd.write (' False Negatives: {:10d}\n'.format(total_false_negatives))
fd.write (' NRI Score: {:0.4f}\n'.format(nri_score))
def EvaluateSkeletons(meta_filename):
data = ReadMetaData(meta_filename)
# make sure a results folder is specified
assert (not data.EvaluationDirectory() == None)
# read in statistics about this data set
statistics_directory = '{}/statistics'.format(data.TempDirectory())
statistics_filename = '{}/combined-statistics.pickle'.format(statistics_directory)
statistics = ReadPickledData(statistics_filename)
label_volumes = statistics['label_volumes']
total_volume = 0
total_thinned_skeleton_length = 0
total_refined_skeleton_length = 0
nlabels = 0
# get the output filename
evaluation_directory = data.EvaluationDirectory()
if not os.path.exists(evaluation_directory):
os.makedirs(evaluation_directory, exist_ok=True)
output_filename = '{}/skeleton-results.txt'.format(evaluation_directory)
fd = open(output_filename, 'w')
for label in sorted(label_volumes):
# read pre-refinement skeleton
thinning_filename = '{}/skeletons/{:016d}.pts'.format(data.TempDirectory(), label)
# skip files that do not exist (no synapses, e.g.)
if not os.path.exists(thinning_filename): continue
thinned_skeletons_global_pts, _ = ReadPtsFile(data, thinning_filename)
thinned_skeletons = thinned_skeletons_global_pts[label]
refined_filename = '{}/skeletons/{:016d}.pts'.format(data.SkeletonOutputDirectory(), label)
refined_skeletons_global_pts, _ = ReadPtsFile(data, refined_filename)
refined_skeletons = refined_skeletons_global_pts[label]
# get the volume and total remaining voxels
volume = label_volumes[label]
thinned_skeleton_length = len(thinned_skeletons)
refined_skeleton_length = len(refined_skeletons)
# update the variables that aggregate all labels
total_volume += volume
total_thinned_skeleton_length += thinned_skeleton_length
total_refined_skeleton_length += refined_skeleton_length
# calculate the percent and reduction of total voxels remaining
thinned_remaining_percent = 100 * thinned_skeleton_length / volume
thinning_reduction_factor = volume / thinned_skeleton_length
refined_remaining_percent = 100 * refined_skeleton_length / thinned_skeleton_length
refinement_reduction_factor = thinned_skeleton_length / refined_skeleton_length
# caluclate the total percent/reduction after all steps
total_skeleton_percent = 100 * refined_skeleton_length / volume
total_skeleton_reduction = volume / refined_skeleton_length
print ('Label: {}'.format(label))
print (' Input Volume: {:10d}'.format(volume))
print (' Topological Thinning: {:10d} ({:05.2f}%) {:10.2f}x'.format(thinned_skeleton_length, thinned_remaining_percent, thinning_reduction_factor))
print (' Skeleton Refinement: {:10d} ({:05.2f}%) {:10.2f}x'.format(refined_skeleton_length, refined_remaining_percent, refinement_reduction_factor))
print (' Total: ({:05.2f}%) {:10.2f}x'.format(total_skeleton_percent, total_skeleton_reduction))
fd.write ('Label: {}\n'.format(label))
fd.write (' Input Volume: {:10d}\n'.format(volume))
fd.write (' Topological Thinning: {:10d} ({:05.2f}%) {:10.2f}x\n'.format(thinned_skeleton_length, thinned_remaining_percent, thinning_reduction_factor))
fd.write (' Skeleton Refinement: {:10d} ({:05.2f}%) {:10.2f}x\n'.format(refined_skeleton_length, refined_remaining_percent, refinement_reduction_factor))
fd.write (' Total: ({:05.2f}%) {:10.2f}x\n'.format(total_skeleton_percent, total_skeleton_reduction))
nlabels += 1
# calculate the percent and reduction of total voxels remaining
thinned_remaining_percent = 100 * total_thinned_skeleton_length / total_volume
thinning_reduction_factor = total_volume / total_thinned_skeleton_length
refined_remaining_percent = 100 * total_refined_skeleton_length / total_thinned_skeleton_length
refinement_reduction_factor = total_thinned_skeleton_length / total_refined_skeleton_length
# caluclate the total percent/reduction after all steps
total_skeleton_percent = 100 * total_refined_skeleton_length / total_volume
total_skeleton_reduction = total_volume / total_refined_skeleton_length
print ('Input Volume: {:10d}'.format(total_volume))
print ('Topological Thinning: {:10d} ({:05.2f}%) {:10.2f}x'.format(total_thinned_skeleton_length, thinned_remaining_percent, thinning_reduction_factor))
print ('Skeleton Refinement: {:10d} ({:05.2f}%) {:10.2f}x'.format(total_refined_skeleton_length, refined_remaining_percent, refinement_reduction_factor))
print ('Total: ({:05.2f}%) {:10.2f}x'.format(total_skeleton_percent, total_skeleton_reduction))
print ('Average Skeleton: {:0.0f}'.format(total_refined_skeleton_length / nlabels))
fd.write ('Input Volume: {:10d}\n'.format(total_volume))
fd.write ('Topological Thinning: {:10d} ({:05.2f}%) {:10.2f}x\n'.format(total_thinned_skeleton_length, thinned_remaining_percent, thinning_reduction_factor))
fd.write ('Skeleton Refinement: {:10d} ({:05.2f}%) {:10.2f}x\n'.format(total_refined_skeleton_length, refined_remaining_percent, refinement_reduction_factor))
fd.write ('Total: ({:05.2f}%) {:10.2f}x\n'.format(total_skeleton_percent, total_skeleton_reduction))
fd.write ('Average Skeleton: {:0.0f}'.format(total_refined_skeleton_length / nlabels))
# close the file
fd.close()
def MapSynapsesAndSurfaces(input_meta_filename, output_meta_filename):
# read in the meta data files
input_data = ReadMetaData(input_meta_filename)
output_data = ReadMetaData(output_meta_filename)
input_synapse_directory = input_data.SynapseDirectory()
output_synapse_directory = output_data.SynapseDirectory()
# create the output synapse directory if it does not exist
if not os.path.exists(output_synapse_directory):
os.makedirs(output_synapse_directory, exist_ok=True)
# read in all of the input syanpses
input_synapses = {}
for iz in range(input_data.StartZ(), input_data.EndZ()):
for iy in range(input_data.StartY(), input_data.EndY()):
for ix in range(input_data.StartX(), input_data.EndX()):
input_synapse_filename = '{}/{:04d}z-{:04d}y-{:04d}x.pts'.format(input_synapse_directory, iz, iy, ix)
# ignore the local points
global_pts, _ = ReadPtsFile(input_data, input_synapse_filename)
for label in global_pts:
if not label in input_synapses:
input_synapses[label] = []
# add the array from this block to the input synapses
input_synapses[label] += global_pts[label]
# create an output dictionary of synapses per block
output_synapses_per_block = {}
# iterate over all output blocks
for iz in range(output_data.StartZ(), output_data.EndZ()):
for iy in range(output_data.StartY(), output_data.EndY()):
for ix in range(output_data.StartX(), output_data.EndX()):
# create empty synapses_per_block dictionaries whose keys will be labels
output_synapses_per_block[(iz, iy, ix)] = {}
# for every label, go through the discovered synapses from the input data
for label in input_synapses.keys():
input_synapses_per_label = input_synapses[label]
for input_global_index in input_synapses_per_label:
# the global iz, iy, ix coordinates remain the same across blocks
global_iz, global_iy, global_ix = input_data.GlobalIndexToIndices(input_global_index)
# get the new block from the global coordinates
output_block_iz = global_iz // output_data.BlockZLength()
output_block_iy = global_iy // output_data.BlockYLength()
output_block_ix = global_ix // output_data.BlockXLength()
if not label in output_synapses_per_block[(output_block_iz, output_block_iy, output_block_ix)]:
output_synapses_per_block[(output_block_iz, output_block_iy, output_block_ix)][label] = []
# get the new global index
output_global_index = output_data.GlobalIndicesToIndex(global_iz, global_iy, global_ix)
output_synapses_per_block[(output_block_iz, output_block_iy, output_block_ix)][label].append(output_global_index)
# write all of the synapse block files
output_synapses_directory = output_data.SynapseDirectory()
for iz in range(output_data.StartZ(), output_data.EndZ()):
for iy in range(output_data.StartY(), output_data.EndY()):
for ix in range(output_data.StartX(), output_data.EndX()):
output_synapse_filename = '{}/{:04d}z-{:04d}y-{:04d}x.pts'.format(output_synapse_directory, iz, iy, ix)
# write the pts file (use global indices)
WritePtsFile(output_data, output_synapse_filename, output_synapses_per_block[(iz, iy, ix)], input_local_indices = False)
# get the input/output directories for the surfaces
input_surfaces_directory = input_data.SurfacesDirectory()
output_surfaces_directory = output_data.SurfacesDirectory()
# create the output synapse directory if it does not exist
if not os.path.exists(output_surfaces_directory):
os.makedirs(output_surfaces_directory, exist_ok=True)
# iterate over all labels
for label in range(1, input_data.NLabels()):
start_time = time.time()
# skip over labels that do not exist
input_surface_filename = '{}/{:016d}.pts'.format(input_surfaces_directory, label)
if not os.path.exists(input_surface_filename): continue
# read in the input global points
input_global_points, _ = ReadPtsFile(input_data, input_surface_filename)
# create an empty dictionary for the output points
output_global_points = {}
output_global_points[label] = []
for input_global_index in input_global_points[label]:
# the global iz, iy, ix coordinates remain the same across blocks
global_iz, global_iy, global_ix = input_data.GlobalIndexToIndices(input_global_index)
# get the new global index
output_global_index = output_data.GlobalIndicesToIndex(global_iz, global_iy, global_ix)
output_global_points[label].append(output_global_index)
# write the new surface filename
output_surface_filename = '{}/{:016d}.pts'.format(output_surfaces_directory, label)
WritePtsFile(output_data, output_surface_filename, output_global_points, input_local_indices = False)
print ('Completed label {} in {:0.2f} seconds'.format(label, time.time() - start_time))
def ConvertSynapsesAndProject(meta_filename, input_synapse_directory, xyz,
conversion_rate):
data = ReadMetaData(meta_filename)
resolution = data.Resolution()
# create an empty set of synapses
synapses = {}
# iterate over all labels
for label in range(1, data.NLabels()):
# read the surfaces for this label
surface_filename = '{}/{:016d}.pts'.format(data.SurfacesDirectory(),
label)
# some surfaces (i.e., labels) will not exist in the volume
if not os.path.exists(surface_filename): continue
# read in the surface points, ignore the local coordinates
surfaces, _ = ReadPtsFile(data, surface_filename)
surface = surfaces[label]
npts = len(surface)
surface_point_cloud = np.zeros((npts, 3), dtype=np.int32)
for index, iv in enumerate(surface):
iz, iy, ix = data.GlobalIndexToIndices(iv)
surface_point_cloud[index, :] = (iz * resolution[OR_Z],
iy * resolution[OR_Y],
ix * resolution[OR_X])
# create an empty array for the synapses
synapses[label] = []
projected = 0
missed = 0
# read in the original synapses
input_synapse_filename = '{}/syn_{:04}.txt'.format(
input_synapse_directory, label)
if os.path.exists(input_synapse_filename):
with open(input_synapse_filename, 'r') as fd:
for line in fd:
# separate the line into coordinates
coordinates = line.strip().split()
if xyz:
ix = round(int(coordinates[0]) / conversion_rate[OR_X])
iy = round(int(coordinates[1]) / conversion_rate[OR_Y])
iz = round(int(coordinates[2]) / conversion_rate[OR_Z])
else:
iz = round(int(coordinates[0]) / conversion_rate[OR_Z])
iy = round(int(coordinates[1]) / conversion_rate[OR_Y])
ix = round(int(coordinates[2]) / conversion_rate[OR_X])
# create a 2D vector for this point
vec = np.zeros((1, 3), dtype=np.int32)
vec[0, :] = (iz * resolution[OR_Z], iy * resolution[OR_Y],
ix * resolution[OR_X])
closest_point = surface[scipy.spatial.distance.cdist(
surface_point_cloud, vec).argmin()]
closest_iz, closest_iy, closest_ix = data.GlobalIndexToIndices(
closest_point)
deltaz = resolution[OR_Z] * (iz - closest_iz)
deltay = resolution[OR_Y] * (iy - closest_iy)
deltax = resolution[OR_X] * (ix - closest_ix)
distance = math.sqrt(deltaz * deltaz + deltay * deltay +
deltax * deltax)
# skip distances that are clearly off (over 200 nanometers)
max_deviation = 800
if distance < max_deviation:
# add to the list of valid synapses
synapses[label].append(closest_point)
projected += 1
else:
missed += 1
print('Synapses within {} nanometers from surface: {}'.format(
max_deviation, projected))
print('Synapses over {} nanometers from surface: {}'.format(
max_deviation, missed))
# divide all synapses into blocks
synapses_per_block = {}
# iterate over all blocks
for iz in range(data.StartZ(), data.EndZ()):
for iy in range(data.StartY(), data.EndY()):
for ix in range(data.StartX(), data.EndX()):
# create empty synapses_per_block dictionaries whose keys will be labels
synapses_per_block[(iz, iy, ix)] = {}
# for every label, iterate over the discovered synapses
for label in synapses.keys():
synapses_per_label = synapses[label]
# iterate over all of the projected synapses
for global_index in synapses_per_label:
global_iz, global_iy, global_ix = data.GlobalIndexToIndices(
global_index)
block_iz = global_iz // data.BlockZLength()
block_iy = global_iy // data.BlockYLength()
block_ix = global_ix // data.BlockXLength()
# create the array for this label per block if it does not exist
if not label in synapses_per_block[(block_iz, block_iy, block_ix)]:
synapses_per_block[(block_iz, block_iy, block_ix)][label] = []
synapses_per_block[(block_iz, block_iy,
block_ix)][label].append(global_index)
# write all of the synapse block files
synapse_directory = data.SynapseDirectory()
if not os.path.exists(synapse_directory):
os.makedirs(synapse_directory, exist_ok=True)
for iz in range(data.StartZ(), data.EndZ()):
for iy in range(data.StartY(), data.EndY()):
for ix in range(data.StartX(), data.EndX()):
synapse_filename = '{}/{:04d}z-{:04d}y-{:04d}x.pts'.format(
synapse_directory, iz, iy, ix)
# write the pts file (use global indices)
WritePtsFile(data,
synapse_filename,
synapses_per_block[(iz, iy, ix)],
input_local_indices=False)