def create_points(animal, layer, url_id, create): fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) # Get src points data df = sqlController.get_point_dataframe(url_id) df = df[df['Layer'] == layer] df.sort_values(by=['Section', 'X', 'Y'], inplace=True) if create: records = df[['X', 'Y', 'Section']].to_records(index=False) coordinates = list(records) width = sqlController.scan_run.width height = sqlController.scan_run.height sections = sqlController.get_section_count(animal) layer = str(layer).replace(' ', '_') OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, layer) if os.path.exists(OUTPUT_DIR): shutil.rmtree(OUTPUT_DIR) os.makedirs(OUTPUT_DIR, exist_ok=True) info = os.path.join(DATA_PATH, 'atlas_data', 'points', 'info') outfile = os.path.join(OUTPUT_DIR, 'info') with open(info, 'r+') as rf: data = json.load(rf) data['upper_bound'] = [width, height, sections] # <--- add `id` value. rf.seek(0) # <--- should reset file position to the beginning. with open(outfile, 'w') as wf: json.dump(data, wf, indent=4) spatial_dir = os.path.join(OUTPUT_DIR, 'spatial0') os.makedirs(spatial_dir) total_count = len( coordinates) # coordinates is a list of tuples (x,y,z) with open(os.path.join(spatial_dir, '0_0_0.gz'), 'wb') as unzippedfile: buf = struct.pack('<Q', total_count) pt_buf = b''.join( struct.pack('<3f', x, y, z) for (x, y, z) in coordinates) buf += pt_buf id_buf = struct.pack('<%sQ' % len(coordinates), *range(len(coordinates))) buf += id_buf bufout = gzip.compress(buf) unzippedfile.write(bufout) else: print(df['Layer'].unique()) print(df.head(25))
def test_dir(animal, dir, downsample=True, same_size=False): error = "" #thumbnail resolution ntb is 10400 and min size of DK52 is 16074 #thumbnail resolution thion is 14464 and min size for MD585 is 21954 # so 10000 is a good min size # min size on NTB is 8.8K starting_size = 4000 min_size = starting_size * SCALING_FACTOR * 1000 if downsample: min_size = starting_size sqlController = SqlController(animal) section_count = sqlController.get_section_count(animal) try: files = sorted(os.listdir(dir)) except: return f'{dir} does not exist' if section_count == 0: section_count = len(files) widths = set() heights = set() for f in files: filepath = os.path.join(dir, f) width, height = get_image_size(filepath) widths.add(int(width)) heights.add(int(height)) size = os.path.getsize(filepath) if size < min_size: error += f"{size} is less than min: {min_size} {filepath} \n" # picked 100 as an arbitrary number. the min file count is usually around 380 or so if len(files) > 100: min_width = min(widths) max_width = max(widths) min_height = min(heights) max_height = max(heights) else: min_width = 0 max_width = 0 min_height = 0 max_height = 0 if section_count != len(files): error += f"Number of files in {dir} is incorrect.\n" if min_width != max_width and min_width > 0 and same_size: error += f"Widths are not of equal size, min is {min_width} and max is {max_width}.\n" if min_height != max_height and min_height > 0 and same_size: error += f"Heights are not of equal size, min is {min_height} and max is {max_height}.\n" return error
def create_atlas(animal, create): sql_controller = SqlController(animal) fileLocationManager = FileLocationManager(animal) atlas_name = 'atlasV7' ATLAS_PATH = os.path.join(DATA_PATH, 'atlas_data', atlas_name) OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'atlas') if os.path.exists(OUTPUT_DIR) and create: shutil.rmtree(OUTPUT_DIR) os.makedirs(OUTPUT_DIR, exist_ok=True) resolution = sql_controller.scan_run.resolution SCALE = (10 / resolution) reference_centers = sql_controller.get_centers_dict(animal) structures = sorted(reference_centers.keys()) #src_point_set = np.array([atlas_centers[s] for s in structures]).T atlas_box_scales = np.array([10, 10, 20]) #src_point_set = np.diag(atlas_box_scales) @ src_point_set dst_point_set = np.array([reference_centers[s] for s in structures]).T #####Transform to auto align #x_out = inv(output_scale) @ x #x can be replaced by output_point_set_expected_auto or t_auto. #pprint(all_atlas_centers) #sys.exit() #x_out = np.linalg.inv(output_scale) @ x Rx = np.array([[0.99539957, 0.36001948, 0.01398446], [-0.35951649, 0.99520404, -0.03076857], [-0.02361111, 0.02418234, 1.05805842]]) tx = np.array([[19186.25529129], [9825.28539829], [78.18301303]]) R, t = align_atlas(reference_centers) rotationpath = os.path.join(ATLAS_PATH, f'atlas2{animal}.rotation.npy') np.save(rotationpath, R) translatepath = os.path.join(ATLAS_PATH, f'atlas2{animal}.translation.npy') np.save(translatepath, t) print(f'resolution: {sql_controller.scan_run.resolution}') print(f'width: {sql_controller.scan_run.width}') print(f'height: {sql_controller.scan_run.height}') box_w = sql_controller.scan_run.width / SCALE # 10 mum scale box_h = sql_controller.scan_run.height / SCALE # 10 mum scale box_z = sql_controller.get_section_count(animal) # 20 mum scale output_scale = np.diagflat([0.325, 0.325, 20]) atlasV7_volume = np.zeros((int(box_h), int(box_w), int(box_z)), dtype=np.uint8) print('Shape of atlas volume', atlasV7_volume.shape) debug = True atlas_centers_volumes = get_atlas_centers_hardcodeddata() for structure, (source_point, volume) in sorted(atlas_centers_volumes.items()): print(str(structure).ljust(7), end=": ") results = (R @ source_point.T + t.T) # transform to fit #results = np.linalg.inv(output_scale) @ results x = results[0][0] / SCALE # new x y = results[0][1] / SCALE # new y z = results[0][2] # z x = x - volume.shape[1] / 2 y = y - volume.shape[0] / 2 x_start = int(round(x)) y_start = int(round(y)) z_start = int(z - volume.shape[2] / 4) x_end = int(round(x_start + volume.shape[1])) y_end = int(round(y_start + volume.shape[0])) z_end = int(round(z_start + (volume.shape[2] + 1) // 2)) if debug: #print('volume shape', volume.shape, end=" ") """ print('COM row', str(int(y)).rjust(4), 'mid col', str(int(x)).rjust(4), 'mid z', str(int(z)).rjust(4), end=" ") """ print('Row range', str(y_start).rjust(4), str(y_end).rjust(4), 'col range', str(x_start).rjust(4), str(x_end).rjust(4), 'z range', str(z_start).rjust(4), str(z_end).rjust(4), end=" ") z_indices = [z for z in range(volume.shape[2]) if z % 2 == 0] volume = volume.astype(np.uint8) volume = volume[:, :, z_indices] volume = np.swapaxes(volume, 0, 1) try: atlasV7_volume[y_start:y_end, x_start:x_end, z_start:z_end] += volume except ValueError as ve: print('Bad fit', end=" ") print() print('Shape of downsampled atlas volume', atlasV7_volume.shape) resolution = 10000 print('Resolution at', resolution) if create: atlasV7_volume = np.rot90(atlasV7_volume, axes=(0, 1)) atlasV7_volume = np.fliplr(atlasV7_volume) atlasV7_volume = np.flipud(atlasV7_volume) atlasV7_volume = np.fliplr(atlasV7_volume) offset = [0, 0, 0] ng = NumpyToNeuroglancer(atlasV7_volume, [resolution, resolution, 20000], offset=offset) ng.init_precomputed(OUTPUT_DIR) ng.add_segment_properties(get_segment_properties()) ng.add_downsampled_volumes() ng.add_segmentation_mesh() #outpath = os.path.join(ATLAS_PATH, f'{atlas_name}.tif') #io.imsave(outpath, atlasV7_volume.astype(np.uint8)) end = timer() print(f'Finito! Program took {end - start} seconds')