def crop_negative_patches_from_oct(oct_path='/nrs/mouselight/SAMPLES/2017-09-25-padded', coord_list='/groups/mousebrainmicro/mousebrainmicro/users/base/AnnotationData/h5repo/negative_sample_coordinates.txt', patch_half_size=(25, 25, 25)): params = util.readParameterFile(parameterfile=oct_path+"/calculated_parameters.jl") scale = 1/1000 with open(coord_list, 'r') as f: um = f.readlines() um = np.array([eval(x.strip()) for x in um]) # um, edges, R, offset, scale, header = util.readSWC(swcfile=coord_list,scale=1) # to fix the bug in Janelia Workstation um = um + params['vixsize']/2 nm =um/scale xyz = util.um2pix(nm,params['A']).T ## neg_patches = oct_crop(params, oct_path, xyz, patch_half_size=patch_half_size) return np.stack(neg_patches,axis=0)
def crop_from_render(data_fold, input_swc, output_folder, output_swc_name, output_h5_name, scale, cast2vox=True): output_swc_file = os.path.join(output_folder, output_swc_name) output_h5_file = os.path.join(output_folder, output_h5_name) params = util.readParameterFile(parameterfile=data_fold + "/calculated_parameters.jl") # check if input argument is file or folder if os.path.isfile(input_swc): inputfolder, swc_name_w_ext = os.path.split(input_swc) nm, edges, R, offset, scale, header = util.readSWC( swcfile=os.path.join(inputfolder, swc_name_w_ext), scale=scale) elif os.path.isdir(input_swc): inputfolder = input_swc nm, edges, R = util.appendSWCfolder( inputfolder, scale=scale) # somewhat redundant but cleaner nm_, edges_, R_, filenames, header = util.readSWCfolder(inputfolder, scale=scale) if cast2vox: # to fix the bug in Janelia Workstation nm = nm + params['vixsize'] / scale / 2 xyz_ori = util.um2pix(nm, params['A']).T # LVV BUG FIX: if source is JW, fix coordinates xyz_correct = xyz_LVV-[1 1 0] # if any([re.findall('Janelia Workstation Large Volume Viewer', lines) for lines in header]): # xyz=xyz-[1,1,0] # upsample xyz if needed diff_xyz = np.diff(xyz_ori, axis=0) norm_xyz = np.sqrt(np.sum(diff_xyz**2, axis=1)) if np.mean(norm_xyz) > 5: xyz = util.upsampleSWC(xyz_ori, edges, sp=10) else: xyz = nm if False: octpath, xres = improc.xyz2oct(xyz, params) else: depthextend = 3 params_p1 = params.copy() params_p1["nlevels"] = params_p1["nlevels"] + depthextend params_p1["leafshape"] = params_p1["leafshape"] / (2**depthextend) octpath, xres = improc.xyz2oct(xyz, params_p1) octpath_cover = np.unique(octpath, axis=0) gridlist_cover = improc.oct2grid(octpath_cover) octpath_dilated, gridlist_dilated = improc.dilateOct(octpath_cover) #### second pass (somewhat heuristic, helps with cropping later on) octpath_dilated, gridlist_dilated = improc.dilateOct(octpath_dilated) # octpath_dilated,gridlist_dilated = improc.dilateOct(octpath_dilated) # octpath_dilated,gridlist_dilated = improc.dilateOct(octpath_dilated) depthBase = params["nlevels"].astype(int) depthFull = params_p1["nlevels"].astype(int) tileSize = params["leafshape"].astype(int) leafSize = params_p1["leafshape"].astype(int) tilelist = improc.chunklist(octpath_dilated, depthBase) #1..8 tileids = list(tilelist.keys()) # base on bounding box (results in cropped output volume) # gridReference = np.min(gridlist_dilated, axis=0) # gridSize = np.max(gridlist_dilated, axis=0) - gridReference +1 # base on initial image gridReference = np.array((0, 0, 0)) gridSize = tileSize * (2**(depthBase)) / leafSize volReference = gridReference * leafSize outVolumeSize = np.append(gridSize * leafSize, 2) #append color channel chunksize = np.append(leafSize, 2) setting = dict() setting['volSize'] = outVolumeSize setting['chunkSize'] = tuple(chunksize) setting['depthBase'] = depthBase setting['depthFull'] = depthFull setting['tileSize'] = tileSize setting['leafSize'] = leafSize setting['volReference'] = volReference setting['compression'] = "gzip" setting['compression_opts'] = 9 setting['dtype'] = 'uint16' setting['type'] = 'h5' # save into cropped swc xyz_shifted = xyz_ori - volReference if xyz_shifted.ndim < 2: xyz_shifted = xyz_shifted[ None, :] # extend dimension to prevent r < h cases # TODO: fix edge upsampling before dumping swc file # with open(output_swc_file,'w') as fswc: # for iter,txt in enumerate(xyz_shidefaultDataPlaceholderfted): # fswc.write('{:.0f} {:.0f} {:.4f} {:.4f} {:.4f} {:.2f} {:.0f}\n'.format(edges[iter,0].__int__(),1,txt[0]-1,txt[1],txt[2],1,edges[iter,1].__int__())) # dump into file with h5py.File(output_h5_file, "w") as f: dset_swc = f.create_dataset("reconstruction", (xyz_shifted.shape[0], 7), dtype='f') for iter, xyz_ in enumerate(xyz_shifted): dset_swc[iter, :] = np.array([ edges[iter, 0].__int__(), 1, xyz_[0], xyz_[1], xyz_[2], 1.0, edges[iter, 1].__int__() ]) if os.path.isdir(input_swc): # dump into file with h5py.File(output_h5_file, "a") as f: swc_group = f.create_group("swc_files") f_swc_original = swc_group.create_group("original") for it, swcname in enumerate(filenames): numrows = R_[it].shape[0] dset_swc = f_swc_original.create_dataset(swcname, (numrows, 7), dtype='f') edges_swc = edges_[it] nm_swc = nm_[it] for iter, xyz_ in enumerate(nm_swc): dset_swc[iter, :] = np.array([ edges_swc[iter, 0].__int__(), 1, xyz_[0], xyz_[1], xyz_[2], 1.0, edges_swc[iter, 1].__int__() ]) f_swc_shifted = swc_group.create_group("shifted") for it, swcname in enumerate(filenames): numrows = R_[it].shape[0] dset_swc = f_swc_shifted.create_dataset(swcname, (numrows, 7), dtype='f') edges_swc = edges_[it] nm_swc = nm_[it] nm_swc = nm_swc + params['vixsize'] / scale / 2 xyz_ori = util.um2pix(nm_swc, params['A']).T nm_swc = xyz_ori - volReference for iter, xyz_ in enumerate(nm_swc): dset_swc[iter, :] = np.array([ edges_swc[iter, 0].__int__(), 1, xyz_[0], xyz_[1], xyz_[2], 1.0, edges_swc[iter, 1].__int__() ]) dump = util.dumper(data_fold, output_h5_file, setting, tilelist=tilelist) dump.write()
def crop_from_render(render_folder_name, input_swc_file_or_folder_name, output_folder_name, output_volume_file_name, do_use_simple_for_loop=False): output_volume_file_path = os.path.join(output_folder_name, output_volume_file_name) params = util.readParameterFile(parameterfile=render_folder_name + "/calculated_parameters.jl") tile_level_count = params["nlevels"].astype(int) tile_shape = params["leafSize"].astype(int) origin_um = params["origin"] spacing_um = params["spacing"] full_volume_shape = tile_shape * 2**tile_level_count # check if input argument is file or folder if os.path.isfile(input_swc_file_or_folder_name): inputfolder, swc_name_w_ext = os.path.split( input_swc_file_or_folder_name) xyz_um, edges, R, offset, scale, header = util.readSWC( os.path.join(inputfolder, swc_name_w_ext)) elif os.path.isdir(input_swc_file_or_folder_name): inputfolder = input_swc_file_or_folder_name xyz_um, edges, R = util.appendSWCfolder( inputfolder) # somewhat redundant but cleaner xyz_um_, edges_, R_, filenames, header = util.readSWCfolder( inputfolder) else: raise RuntimeError('%s does not seem to be a file, nor a folder' % input_swc_file_or_folder_name) # Convert swc coords to voxels xyz_in_voxels = util.um2pix(xyz_um, origin_um, spacing_um) # Each tile in the rendered image will itself be 'octreed' into a set of 'leafs' # depthextend tells now many octree levels there will be within each tile extra_level_count = 3 leaf_level_count = tile_level_count + extra_level_count leaf_shape = (tile_shape / (2**extra_level_count)).astype(int) octpath, xres = improc.ijk2oct(xyz_in_voxels, leaf_level_count, leaf_shape) #depthFull = params_p1["nlevels"].astype(int) #leaf_shape = params_p1["leafshape"].astype(int) swc_base_name = os.path.basename(input_swc_file_or_folder_name) tile_list_pickle_file_name = '%s-tile-list.pickle' % swc_base_name tile_list_pickle_file_path = os.path.join(output_folder_name, tile_list_pickle_file_name) try: tile_hash = pickle.load(open(tile_list_pickle_file_path, 'rb')) print('Loaded tile list from memo file') except os.error: octpath_cover = np.unique(octpath, axis=0) gridlist_cover = improc.oct2grid(octpath_cover) print('About to start dilation...') #octpath_dilated = octpath_cover.copy() desired_carve_out_half_diagonal_as_scalar = 512 # in x,y. z will be different desired_carve_out_half_diagonal = desired_carve_out_half_diagonal_as_scalar * np.array( [1.0, 1.0, spacing_um[0] / spacing_um[2]]) #dilation_count = 8 dilation_count = np.max( np.ceil( desired_carve_out_half_diagonal.astype(float) / leaf_shape.astype(float))).astype(int).item() # should be enough to get about a 512 vx cube around each swc centerpoint # (except 4x less in z, b/c axial rez is less) octpath_dilated, junk = improc.dilateOct(octpath_cover, dilation_count) #for dilation_index in range(dilation_count): # print('Finished dilation iteration %d of %d' % (dilation_index+1, dilation_count)) print('Done with dilation!') tile_hash = improc.chunklist(octpath_dilated, tile_level_count) #1..8 pickle.dump(tile_hash, open(tile_list_pickle_file_path, 'wb')) #tileids = list(tile_hash.keys()) # base on bounding box (results in cropped output volume) # gridReference = np.min(gridlist_dilated, axis=0) # gridSize = np.max(gridlist_dilated, axis=0) - gridReference +1 # base on initial image #gridReference = np.array((0,0,0)) #gridSize = full_volume_shape/leaf_shape # # 3-array, number of leaves in each dimension to make up the full volume #volReference = gridReference*leaf_shape #full_volume_shape_including_color_channel = np.append(full_volume_shape, 2) # append color channel #chunk_shape_including_color_channel = np.append(leaf_shape, 2) #chunk_shape_including_color_channel = np.append(tile_shape, 2) # setting = dict() # setting['volSize'] = full_volume_shape # setting['chunkSize'] = chunksize # setting['depthBase'] = tile_level_count # setting['depthFull'] = leaf_level_count # setting['tileSize'] = tile_shape # setting['leaf_shape'] = leaf_shape # # setting['dtype'] = 'uint16' output_file_extension = os.path.splitext(output_volume_file_name)[1] if output_file_extension == '.h5': output_file_type = 'h5' compression_method = "gzip" compression_options = 9 elif output_file_extension == '.n5': output_file_type = 'n5' compression_method = "gzip" compression_options = {'level': 9} elif output_file_extension == '.zarr': output_file_type = 'zarr' compression_method = "blosc" compression_options = {} else: raise RuntimeError('Don' 't recognize the output file extension %s' % output_file_extension) # Finally, write the voxel carved data to disk color_channel_count = 2 util.dump_write(render_folder_name, full_volume_shape, 'uint16', color_channel_count, output_volume_file_path, tile_hash, leaf_level_count, tile_level_count, compression_method, compression_options, output_file_type, do_use_simple_for_loop)
#sample_point_xyz_centered = np.array([-501.96361500032071, -1013.1980690008477, -4455.3761980023555]) # G-040 centerpoint 33650 #n5_file_path = '/nrs/mouselight/cluster/navigator-output/2018-10-01-tile-chunks-on-cluster/consensus-neurons-with-machine-centerpoints-labelled-as-swcs-carved.n5' #offset = np.array([0, 0 ,0]) #sample_point_xyz_centered = np.array([75148.9, 15244.3, 31761.2]) # G-040 centerpoint copied from WS n5_file_path = '/nrs/mouselight/cluster/navigator-output/2018-10-01-tile-chunks-on-cluster/consensus-neurons-with-machine-centerpoints-labelled-as-swcs-carved.n5' #offset = np.array([73814.192263002085, 17366.654304000425, 36205.867519004452]) #sample_point_xyz_centered = np.array([479.90134299792408, 436.8809699995727, -1025.1585660044511]) # G-017 centerpoint 6320 voxel_center_in_swc = np.array([74447.743653, 17765.863807, 34985.740732]) # this point survives import and export unchanged, and is shown as a voxel center in x-y # it is also s.t. (it - origin_jaws)/spacing is very nearly integral #[origin, spacing] = load_transform_txt(fullfile(octree_path, 'transform.txt')) ; params = util.readParameterFile( parameterfile=os.path.join(octree_path, 'calculated_parameters.jl')) tile_level_count = params["nlevels"].astype(int) tile_shape = params["leafSize"].astype(int) #origin = params["origin"] spacing = params["spacing"] origin_jaws = np.array( [69445.01944245792, 12917.29486937314, 30198.96941474185]) origin = origin_jaws sample_point_xyz = voxel_center_in_swc # this is a voxel center channel_index = 0 infinity_norm_radius = 4 # we want a cube of diameter twice this, in radial voxel units real_sample_point_ijk = real_zero_based_ijk_from_xyz(sample_point_xyz, origin, spacing) sample_point_ijk = zero_based_ijk_from_xyz(sample_point_xyz, origin, spacing)
from util import writeResultsToFile import matplotlib import sys if __name__ == '__main__': print("Passed Parameters: ", len(sys.argv)) print(str(sys.argv)) '___LOAD DATA___' test_folder_name = 'tests/' if (len(sys.argv) > 1): folder_name = str(sys.argv[1]) + '/' else: folder_name = '0/' path = test_folder_name + folder_name training_episodes = readParameterFile(path) cost = readTrainLoss(path) training_rewards = readTrainReward(path) testing_rewards = readTestReward(path) #min_rewards = readTestMinReward("tests/test_min_rewards_1.csv") #max_rewards = readTestMaxReward("tests/test_max_rewards_1.csv") '___PLOT___' #matplotlib.rcParams.update({'font.size':18}) #plotCostFunction(cost) #plotTrainingRewards(training_rewards) #plotTestRewards(testing_rewards) #plotTestResults(testing_rewards, min_rewards, max_rewards, training_episodes) '___STATISTIC___' sumr = 0