def create_points(animal, layer, url_id, create):
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
# Get src points data
df = sqlController.get_point_dataframe(url_id)
df = df[df['Layer'] == layer]
df.sort_values(by=['Section', 'X', 'Y'], inplace=True)
if create:
records = df[['X', 'Y', 'Section']].to_records(index=False)
coordinates = list(records)
width = sqlController.scan_run.width
height = sqlController.scan_run.height
sections = sqlController.get_section_count(animal)
layer = str(layer).replace(' ', '_')
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, layer)
if os.path.exists(OUTPUT_DIR):
shutil.rmtree(OUTPUT_DIR)
os.makedirs(OUTPUT_DIR, exist_ok=True)
info = os.path.join(DATA_PATH, 'atlas_data', 'points', 'info')
outfile = os.path.join(OUTPUT_DIR, 'info')
with open(info, 'r+') as rf:
data = json.load(rf)
data['upper_bound'] = [width, height,
sections] # <--- add `id` value.
rf.seek(0) # <--- should reset file position to the beginning.
with open(outfile, 'w') as wf:
json.dump(data, wf, indent=4)
spatial_dir = os.path.join(OUTPUT_DIR, 'spatial0')
os.makedirs(spatial_dir)
total_count = len(
coordinates) # coordinates is a list of tuples (x,y,z)
with open(os.path.join(spatial_dir, '0_0_0.gz'), 'wb') as unzippedfile:
buf = struct.pack('<Q', total_count)
pt_buf = b''.join(
struct.pack('<3f', x, y, z) for (x, y, z) in coordinates)
buf += pt_buf
id_buf = struct.pack('<%sQ' % len(coordinates),
*range(len(coordinates)))
buf += id_buf
bufout = gzip.compress(buf)
unzippedfile.write(bufout)
else:
print(df['Layer'].unique())
print(df.head(25))
def test_dir(animal, dir, downsample=True, same_size=False):
error = ""
#thumbnail resolution ntb is 10400 and min size of DK52 is 16074
#thumbnail resolution thion is 14464 and min size for MD585 is 21954
# so 10000 is a good min size
# min size on NTB is 8.8K
starting_size = 4000
min_size = starting_size * SCALING_FACTOR * 1000
if downsample:
min_size = starting_size
sqlController = SqlController(animal)
section_count = sqlController.get_section_count(animal)
try:
files = sorted(os.listdir(dir))
except:
return f'{dir} does not exist'
if section_count == 0:
section_count = len(files)
widths = set()
heights = set()
for f in files:
filepath = os.path.join(dir, f)
width, height = get_image_size(filepath)
widths.add(int(width))
heights.add(int(height))
size = os.path.getsize(filepath)
if size < min_size:
error += f"{size} is less than min: {min_size} {filepath} \n"
# picked 100 as an arbitrary number. the min file count is usually around 380 or so
if len(files) > 100:
min_width = min(widths)
max_width = max(widths)
min_height = min(heights)
max_height = max(heights)
else:
min_width = 0
max_width = 0
min_height = 0
max_height = 0
if section_count != len(files):
error += f"Number of files in {dir} is incorrect.\n"
if min_width != max_width and min_width > 0 and same_size:
error += f"Widths are not of equal size, min is {min_width} and max is {max_width}.\n"
if min_height != max_height and min_height > 0 and same_size:
error += f"Heights are not of equal size, min is {min_height} and max is {max_height}.\n"
return error
def create_atlas(animal, create):
sql_controller = SqlController(animal)
fileLocationManager = FileLocationManager(animal)
atlas_name = 'atlasV7'
ATLAS_PATH = os.path.join(DATA_PATH, 'atlas_data', atlas_name)
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'atlas')
if os.path.exists(OUTPUT_DIR) and create:
shutil.rmtree(OUTPUT_DIR)
os.makedirs(OUTPUT_DIR, exist_ok=True)
resolution = sql_controller.scan_run.resolution
SCALE = (10 / resolution)
reference_centers = sql_controller.get_centers_dict(animal)
structures = sorted(reference_centers.keys())
#src_point_set = np.array([atlas_centers[s] for s in structures]).T
atlas_box_scales = np.array([10, 10, 20])
#src_point_set = np.diag(atlas_box_scales) @ src_point_set
dst_point_set = np.array([reference_centers[s] for s in structures]).T
#####Transform to auto align
#x_out = inv(output_scale) @ x
#x can be replaced by output_point_set_expected_auto or t_auto.
#pprint(all_atlas_centers)
#sys.exit()
#x_out = np.linalg.inv(output_scale) @ x
Rx = np.array([[0.99539957, 0.36001948, 0.01398446],
[-0.35951649, 0.99520404, -0.03076857],
[-0.02361111, 0.02418234, 1.05805842]])
tx = np.array([[19186.25529129], [9825.28539829], [78.18301303]])
R, t = align_atlas(reference_centers)
rotationpath = os.path.join(ATLAS_PATH, f'atlas2{animal}.rotation.npy')
np.save(rotationpath, R)
translatepath = os.path.join(ATLAS_PATH, f'atlas2{animal}.translation.npy')
np.save(translatepath, t)
print(f'resolution: {sql_controller.scan_run.resolution}')
print(f'width: {sql_controller.scan_run.width}')
print(f'height: {sql_controller.scan_run.height}')
box_w = sql_controller.scan_run.width / SCALE # 10 mum scale
box_h = sql_controller.scan_run.height / SCALE # 10 mum scale
box_z = sql_controller.get_section_count(animal) # 20 mum scale
output_scale = np.diagflat([0.325, 0.325, 20])
atlasV7_volume = np.zeros((int(box_h), int(box_w), int(box_z)),
dtype=np.uint8)
print('Shape of atlas volume', atlasV7_volume.shape)
debug = True
atlas_centers_volumes = get_atlas_centers_hardcodeddata()
for structure, (source_point,
volume) in sorted(atlas_centers_volumes.items()):
print(str(structure).ljust(7), end=": ")
results = (R @ source_point.T + t.T) # transform to fit
#results = np.linalg.inv(output_scale) @ results
x = results[0][0] / SCALE # new x
y = results[0][1] / SCALE # new y
z = results[0][2] # z
x = x - volume.shape[1] / 2
y = y - volume.shape[0] / 2
x_start = int(round(x))
y_start = int(round(y))
z_start = int(z - volume.shape[2] / 4)
x_end = int(round(x_start + volume.shape[1]))
y_end = int(round(y_start + volume.shape[0]))
z_end = int(round(z_start + (volume.shape[2] + 1) // 2))
if debug:
#print('volume shape', volume.shape, end=" ")
"""
print('COM row',
str(int(y)).rjust(4),
'mid col',
str(int(x)).rjust(4),
'mid z',
str(int(z)).rjust(4),
end=" ")
"""
print('Row range',
str(y_start).rjust(4),
str(y_end).rjust(4),
'col range',
str(x_start).rjust(4),
str(x_end).rjust(4),
'z range',
str(z_start).rjust(4),
str(z_end).rjust(4),
end=" ")
z_indices = [z for z in range(volume.shape[2]) if z % 2 == 0]
volume = volume.astype(np.uint8)
volume = volume[:, :, z_indices]
volume = np.swapaxes(volume, 0, 1)
try:
atlasV7_volume[y_start:y_end, x_start:x_end,
z_start:z_end] += volume
except ValueError as ve:
print('Bad fit', end=" ")
print()
print('Shape of downsampled atlas volume', atlasV7_volume.shape)
resolution = 10000
print('Resolution at', resolution)
if create:
atlasV7_volume = np.rot90(atlasV7_volume, axes=(0, 1))
atlasV7_volume = np.fliplr(atlasV7_volume)
atlasV7_volume = np.flipud(atlasV7_volume)
atlasV7_volume = np.fliplr(atlasV7_volume)
offset = [0, 0, 0]
ng = NumpyToNeuroglancer(atlasV7_volume,
[resolution, resolution, 20000],
offset=offset)
ng.init_precomputed(OUTPUT_DIR)
ng.add_segment_properties(get_segment_properties())
ng.add_downsampled_volumes()
ng.add_segmentation_mesh()
#outpath = os.path.join(ATLAS_PATH, f'{atlas_name}.tif')
#io.imsave(outpath, atlasV7_volume.astype(np.uint8))
end = timer()
print(f'Finito! Program took {end - start} seconds')
