def crop_negative_patches_from_oct(oct_path='/nrs/mouselight/SAMPLES/2017-09-25-padded',
coord_list='/groups/mousebrainmicro/mousebrainmicro/users/base/AnnotationData/h5repo/negative_sample_coordinates.txt',
patch_half_size=(25, 25, 25)):
params = util.readParameterFile(parameterfile=oct_path+"/calculated_parameters.jl")
scale = 1/1000
with open(coord_list, 'r') as f:
um = f.readlines()
um = np.array([eval(x.strip()) for x in um])
# um, edges, R, offset, scale, header = util.readSWC(swcfile=coord_list,scale=1)
# to fix the bug in Janelia Workstation
um = um + params['vixsize']/2
nm =um/scale
xyz = util.um2pix(nm,params['A']).T
##
neg_patches = oct_crop(params, oct_path, xyz, patch_half_size=patch_half_size)
return np.stack(neg_patches,axis=0)
def crop_from_render(data_fold,
input_swc,
output_folder,
output_swc_name,
output_h5_name,
scale,
cast2vox=True):
output_swc_file = os.path.join(output_folder, output_swc_name)
output_h5_file = os.path.join(output_folder, output_h5_name)
params = util.readParameterFile(parameterfile=data_fold +
"/calculated_parameters.jl")
# check if input argument is file or folder
if os.path.isfile(input_swc):
inputfolder, swc_name_w_ext = os.path.split(input_swc)
nm, edges, R, offset, scale, header = util.readSWC(
swcfile=os.path.join(inputfolder, swc_name_w_ext), scale=scale)
elif os.path.isdir(input_swc):
inputfolder = input_swc
nm, edges, R = util.appendSWCfolder(
inputfolder, scale=scale) # somewhat redundant but cleaner
nm_, edges_, R_, filenames, header = util.readSWCfolder(inputfolder,
scale=scale)
if cast2vox:
# to fix the bug in Janelia Workstation
nm = nm + params['vixsize'] / scale / 2
xyz_ori = util.um2pix(nm, params['A']).T
# LVV BUG FIX: if source is JW, fix coordinates xyz_correct = xyz_LVV-[1 1 0]
# if any([re.findall('Janelia Workstation Large Volume Viewer', lines) for lines in header]):
# xyz=xyz-[1,1,0]
# upsample xyz if needed
diff_xyz = np.diff(xyz_ori, axis=0)
norm_xyz = np.sqrt(np.sum(diff_xyz**2, axis=1))
if np.mean(norm_xyz) > 5:
xyz = util.upsampleSWC(xyz_ori, edges, sp=10)
else:
xyz = nm
if False:
octpath, xres = improc.xyz2oct(xyz, params)
else:
depthextend = 3
params_p1 = params.copy()
params_p1["nlevels"] = params_p1["nlevels"] + depthextend
params_p1["leafshape"] = params_p1["leafshape"] / (2**depthextend)
octpath, xres = improc.xyz2oct(xyz, params_p1)
octpath_cover = np.unique(octpath, axis=0)
gridlist_cover = improc.oct2grid(octpath_cover)
octpath_dilated, gridlist_dilated = improc.dilateOct(octpath_cover)
#### second pass (somewhat heuristic, helps with cropping later on)
octpath_dilated, gridlist_dilated = improc.dilateOct(octpath_dilated)
# octpath_dilated,gridlist_dilated = improc.dilateOct(octpath_dilated)
# octpath_dilated,gridlist_dilated = improc.dilateOct(octpath_dilated)
depthBase = params["nlevels"].astype(int)
depthFull = params_p1["nlevels"].astype(int)
tileSize = params["leafshape"].astype(int)
leafSize = params_p1["leafshape"].astype(int)
tilelist = improc.chunklist(octpath_dilated, depthBase) #1..8
tileids = list(tilelist.keys())
# base on bounding box (results in cropped output volume)
# gridReference = np.min(gridlist_dilated, axis=0)
# gridSize = np.max(gridlist_dilated, axis=0) - gridReference +1
# base on initial image
gridReference = np.array((0, 0, 0))
gridSize = tileSize * (2**(depthBase)) / leafSize
volReference = gridReference * leafSize
outVolumeSize = np.append(gridSize * leafSize, 2) #append color channel
chunksize = np.append(leafSize, 2)
setting = dict()
setting['volSize'] = outVolumeSize
setting['chunkSize'] = tuple(chunksize)
setting['depthBase'] = depthBase
setting['depthFull'] = depthFull
setting['tileSize'] = tileSize
setting['leafSize'] = leafSize
setting['volReference'] = volReference
setting['compression'] = "gzip"
setting['compression_opts'] = 9
setting['dtype'] = 'uint16'
setting['type'] = 'h5'
# save into cropped swc
xyz_shifted = xyz_ori - volReference
if xyz_shifted.ndim < 2:
xyz_shifted = xyz_shifted[
None, :] # extend dimension to prevent r < h cases
# TODO: fix edge upsampling before dumping swc file
# with open(output_swc_file,'w') as fswc:
# for iter,txt in enumerate(xyz_shidefaultDataPlaceholderfted):
# fswc.write('{:.0f} {:.0f} {:.4f} {:.4f} {:.4f} {:.2f} {:.0f}\n'.format(edges[iter,0].__int__(),1,txt[0]-1,txt[1],txt[2],1,edges[iter,1].__int__()))
# dump into file
with h5py.File(output_h5_file, "w") as f:
dset_swc = f.create_dataset("reconstruction",
(xyz_shifted.shape[0], 7),
dtype='f')
for iter, xyz_ in enumerate(xyz_shifted):
dset_swc[iter, :] = np.array([
edges[iter, 0].__int__(), 1, xyz_[0], xyz_[1], xyz_[2], 1.0,
edges[iter, 1].__int__()
])
if os.path.isdir(input_swc):
# dump into file
with h5py.File(output_h5_file, "a") as f:
swc_group = f.create_group("swc_files")
f_swc_original = swc_group.create_group("original")
for it, swcname in enumerate(filenames):
numrows = R_[it].shape[0]
dset_swc = f_swc_original.create_dataset(swcname, (numrows, 7),
dtype='f')
edges_swc = edges_[it]
nm_swc = nm_[it]
for iter, xyz_ in enumerate(nm_swc):
dset_swc[iter, :] = np.array([
edges_swc[iter, 0].__int__(), 1, xyz_[0], xyz_[1],
xyz_[2], 1.0, edges_swc[iter, 1].__int__()
])
f_swc_shifted = swc_group.create_group("shifted")
for it, swcname in enumerate(filenames):
numrows = R_[it].shape[0]
dset_swc = f_swc_shifted.create_dataset(swcname, (numrows, 7),
dtype='f')
edges_swc = edges_[it]
nm_swc = nm_[it]
nm_swc = nm_swc + params['vixsize'] / scale / 2
xyz_ori = util.um2pix(nm_swc, params['A']).T
nm_swc = xyz_ori - volReference
for iter, xyz_ in enumerate(nm_swc):
dset_swc[iter, :] = np.array([
edges_swc[iter, 0].__int__(), 1, xyz_[0], xyz_[1],
xyz_[2], 1.0, edges_swc[iter, 1].__int__()
])
dump = util.dumper(data_fold, output_h5_file, setting, tilelist=tilelist)
dump.write()
def crop_from_render(render_folder_name,
input_swc_file_or_folder_name,
output_folder_name,
output_volume_file_name,
do_use_simple_for_loop=False):
output_volume_file_path = os.path.join(output_folder_name,
output_volume_file_name)
params = util.readParameterFile(parameterfile=render_folder_name +
"/calculated_parameters.jl")
tile_level_count = params["nlevels"].astype(int)
tile_shape = params["leafSize"].astype(int)
origin_um = params["origin"]
spacing_um = params["spacing"]
full_volume_shape = tile_shape * 2**tile_level_count
# check if input argument is file or folder
if os.path.isfile(input_swc_file_or_folder_name):
inputfolder, swc_name_w_ext = os.path.split(
input_swc_file_or_folder_name)
xyz_um, edges, R, offset, scale, header = util.readSWC(
os.path.join(inputfolder, swc_name_w_ext))
elif os.path.isdir(input_swc_file_or_folder_name):
inputfolder = input_swc_file_or_folder_name
xyz_um, edges, R = util.appendSWCfolder(
inputfolder) # somewhat redundant but cleaner
xyz_um_, edges_, R_, filenames, header = util.readSWCfolder(
inputfolder)
else:
raise RuntimeError('%s does not seem to be a file, nor a folder' %
input_swc_file_or_folder_name)
# Convert swc coords to voxels
xyz_in_voxels = util.um2pix(xyz_um, origin_um, spacing_um)
# Each tile in the rendered image will itself be 'octreed' into a set of 'leafs'
# depthextend tells now many octree levels there will be within each tile
extra_level_count = 3
leaf_level_count = tile_level_count + extra_level_count
leaf_shape = (tile_shape / (2**extra_level_count)).astype(int)
octpath, xres = improc.ijk2oct(xyz_in_voxels, leaf_level_count, leaf_shape)
#depthFull = params_p1["nlevels"].astype(int)
#leaf_shape = params_p1["leafshape"].astype(int)
swc_base_name = os.path.basename(input_swc_file_or_folder_name)
tile_list_pickle_file_name = '%s-tile-list.pickle' % swc_base_name
tile_list_pickle_file_path = os.path.join(output_folder_name,
tile_list_pickle_file_name)
try:
tile_hash = pickle.load(open(tile_list_pickle_file_path, 'rb'))
print('Loaded tile list from memo file')
except os.error:
octpath_cover = np.unique(octpath, axis=0)
gridlist_cover = improc.oct2grid(octpath_cover)
print('About to start dilation...')
#octpath_dilated = octpath_cover.copy()
desired_carve_out_half_diagonal_as_scalar = 512 # in x,y. z will be different
desired_carve_out_half_diagonal = desired_carve_out_half_diagonal_as_scalar * np.array(
[1.0, 1.0, spacing_um[0] / spacing_um[2]])
#dilation_count = 8
dilation_count = np.max(
np.ceil(
desired_carve_out_half_diagonal.astype(float) /
leaf_shape.astype(float))).astype(int).item()
# should be enough to get about a 512 vx cube around each swc centerpoint
# (except 4x less in z, b/c axial rez is less)
octpath_dilated, junk = improc.dilateOct(octpath_cover, dilation_count)
#for dilation_index in range(dilation_count):
# print('Finished dilation iteration %d of %d' % (dilation_index+1, dilation_count))
print('Done with dilation!')
tile_hash = improc.chunklist(octpath_dilated, tile_level_count) #1..8
pickle.dump(tile_hash, open(tile_list_pickle_file_path, 'wb'))
#tileids = list(tile_hash.keys())
# base on bounding box (results in cropped output volume)
# gridReference = np.min(gridlist_dilated, axis=0)
# gridSize = np.max(gridlist_dilated, axis=0) - gridReference +1
# base on initial image
#gridReference = np.array((0,0,0))
#gridSize = full_volume_shape/leaf_shape
# # 3-array, number of leaves in each dimension to make up the full volume
#volReference = gridReference*leaf_shape
#full_volume_shape_including_color_channel = np.append(full_volume_shape, 2) # append color channel
#chunk_shape_including_color_channel = np.append(leaf_shape, 2)
#chunk_shape_including_color_channel = np.append(tile_shape, 2)
# setting = dict()
# setting['volSize'] = full_volume_shape
# setting['chunkSize'] = chunksize
# setting['depthBase'] = tile_level_count
# setting['depthFull'] = leaf_level_count
# setting['tileSize'] = tile_shape
# setting['leaf_shape'] = leaf_shape
#
# setting['dtype'] = 'uint16'
output_file_extension = os.path.splitext(output_volume_file_name)[1]
if output_file_extension == '.h5':
output_file_type = 'h5'
compression_method = "gzip"
compression_options = 9
elif output_file_extension == '.n5':
output_file_type = 'n5'
compression_method = "gzip"
compression_options = {'level': 9}
elif output_file_extension == '.zarr':
output_file_type = 'zarr'
compression_method = "blosc"
compression_options = {}
else:
raise RuntimeError('Don'
't recognize the output file extension %s' %
output_file_extension)
# Finally, write the voxel carved data to disk
color_channel_count = 2
util.dump_write(render_folder_name, full_volume_shape, 'uint16',
color_channel_count, output_volume_file_path, tile_hash,
leaf_level_count, tile_level_count, compression_method,
compression_options, output_file_type,
do_use_simple_for_loop)
#sample_point_xyz_centered = np.array([-501.96361500032071, -1013.1980690008477, -4455.3761980023555]) # G-040 centerpoint 33650
#n5_file_path = '/nrs/mouselight/cluster/navigator-output/2018-10-01-tile-chunks-on-cluster/consensus-neurons-with-machine-centerpoints-labelled-as-swcs-carved.n5'
#offset = np.array([0, 0 ,0])
#sample_point_xyz_centered = np.array([75148.9, 15244.3, 31761.2]) # G-040 centerpoint copied from WS
n5_file_path = '/nrs/mouselight/cluster/navigator-output/2018-10-01-tile-chunks-on-cluster/consensus-neurons-with-machine-centerpoints-labelled-as-swcs-carved.n5'
#offset = np.array([73814.192263002085, 17366.654304000425, 36205.867519004452])
#sample_point_xyz_centered = np.array([479.90134299792408, 436.8809699995727, -1025.1585660044511]) # G-017 centerpoint 6320
voxel_center_in_swc = np.array([74447.743653, 17765.863807, 34985.740732])
# this point survives import and export unchanged, and is shown as a voxel center in x-y
# it is also s.t. (it - origin_jaws)/spacing is very nearly integral
#[origin, spacing] = load_transform_txt(fullfile(octree_path, 'transform.txt')) ;
params = util.readParameterFile(
parameterfile=os.path.join(octree_path, 'calculated_parameters.jl'))
tile_level_count = params["nlevels"].astype(int)
tile_shape = params["leafSize"].astype(int)
#origin = params["origin"]
spacing = params["spacing"]
origin_jaws = np.array(
[69445.01944245792, 12917.29486937314, 30198.96941474185])
origin = origin_jaws
sample_point_xyz = voxel_center_in_swc # this is a voxel center
channel_index = 0
infinity_norm_radius = 4 # we want a cube of diameter twice this, in radial voxel units
real_sample_point_ijk = real_zero_based_ijk_from_xyz(sample_point_xyz, origin,
spacing)
sample_point_ijk = zero_based_ijk_from_xyz(sample_point_xyz, origin, spacing)
from util import writeResultsToFile
import matplotlib
import sys
if __name__ == '__main__':
print("Passed Parameters: ", len(sys.argv))
print(str(sys.argv))
'___LOAD DATA___'
test_folder_name = 'tests/'
if (len(sys.argv) > 1): folder_name = str(sys.argv[1]) + '/'
else: folder_name = '0/'
path = test_folder_name + folder_name
training_episodes = readParameterFile(path)
cost = readTrainLoss(path)
training_rewards = readTrainReward(path)
testing_rewards = readTestReward(path)
#min_rewards = readTestMinReward("tests/test_min_rewards_1.csv")
#max_rewards = readTestMaxReward("tests/test_max_rewards_1.csv")
'___PLOT___'
#matplotlib.rcParams.update({'font.size':18})
#plotCostFunction(cost)
#plotTrainingRewards(training_rewards)
#plotTestRewards(testing_rewards)
#plotTestResults(testing_rewards, min_rewards, max_rewards, training_episodes)
'___STATISTIC___'
sumr = 0