def test_transfer_task_vanilla(tq, src_cv, transfer_data):
tasks = tc.create_transfer_tasks(src_cv.cloudpath, destpath)
tq.insert_all(tasks)
dest_cv = CloudVolume(destpath)
assert np.all(src_cv[:] == dest_cv[:])
rmsrc()
rmdest()
def test_transfer_task_skip_downsample(tq, src_cv, chunk_size):
tasks = tc.create_transfer_tasks(src_cv.cloudpath,
destpath,
chunk_size=chunk_size,
skip_downsamples=True)
tq.insert_all(tasks)
dest_cv = CloudVolume(destpath)
assert len(dest_cv.scales) == 1
assert np.all(src_cv[:] == dest_cv[:])
rmsrc()
rmdest()
def create_downsamples(animal, channel, suffix, downsample):
fileLocationManager = FileLocationManager(animal)
channel_outdir = f'C{channel}'
first_chunk = calculate_chunks(downsample, 0)
mips = [0, 1, 2, 3, 4, 5, 6, 7]
if downsample:
channel_outdir += 'T'
mips = [0, 1]
outpath = os.path.join(fileLocationManager.neuroglancer_data,
f'{channel_outdir}')
outpath = f'file://{outpath}'
if suffix is not None:
outpath += suffix
channel_outdir += "_rechunkme"
INPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data,
f'{channel_outdir}')
if not os.path.exists(INPUT_DIR):
print(f'DIR {INPUT_DIR} does not exist, exiting.')
sys.exit()
cloudpath = f"file://{INPUT_DIR}"
_, workers = get_cpus()
tq = LocalTaskQueue(parallel=workers)
tasks = tc.create_transfer_tasks(cloudpath,
dest_layer_path=outpath,
chunk_size=first_chunk,
mip=0,
skip_downsamples=True)
tq.insert(tasks)
tq.execute()
#mips = 7 shows good results in neuroglancer
for mip in mips:
cv = CloudVolume(outpath, mip)
chunks = calculate_chunks(downsample, mip)
factors = calculate_factors(downsample, mip)
tasks = tc.create_downsampling_tasks(cv.layer_cloudpath,
mip=mip,
num_mips=1,
factor=factors,
preserve_chunk_size=False,
compress=True,
chunk_size=chunks)
tq.insert(tasks)
tq.execute()
print("Done!")
def test_transfer_task_rechunk(tq, src_cv, transfer_data):
tasks = tc.create_transfer_tasks(src_cv.cloudpath,
destpath,
chunk_size=(50, 50, 50))
tq.insert_all(tasks)
dest_cv = CloudVolume(destpath)
assert len(dest_cv.scales) == 4
assert np.all(src_cv[:] == dest_cv[:])
for mip in range(1, 4):
dest_cv.mip = mip
assert np.all(dest_cv[:] == transfer_data[mip])
rmsrc()
rmdest()
def add_rechunking(self, outpath, downsample, chunks=None):
if self.precomputed_vol is None:
raise NotImplementedError(
'You have to call init_precomputed before calling this function.'
)
cpus, _ = get_cpus()
tq = LocalTaskQueue(parallel=cpus)
outpath = f'file://{outpath}'
if chunks is None:
chunks = calculate_chunks(downsample, 0)
tasks = tc.create_transfer_tasks(self.precomputed_vol.layer_cloudpath,
dest_layer_path=outpath,
chunk_size=chunks,
skip_downsamples=True)
tq.insert(tasks)
tq.execute()
def test_transfer_task_dest_offset(tq, src_cv, transfer_data):
tasks = tc.create_transfer_tasks(src_cv.cloudpath,
destpath,
chunk_size=(50, 50, 50),
dest_voxel_offset=(100, 100, 100))
tq.insert_all(tasks)
dest_cv = CloudVolume(destpath)
assert len(dest_cv.scales) == 4
assert tuple(dest_cv.voxel_offset) == (100, 100, 100)
assert tuple(src_cv.voxel_offset) == (0, 0, 0)
assert np.all(src_cv[:] == dest_cv[:])
for mip in range(1, 4):
dest_cv.mip = mip
assert np.all(dest_cv[:] == transfer_data[mip])
rmsrc()
rmdest()
def test_transfer_task_subset(tq, src_cv, transfer_data):
dest_cv = CloudVolume(destpath, info=copy.deepcopy(src_cv.info))
dest_cv.scale["size"] = [256, 256, 64]
dest_cv.commit_info()
tasks = tc.create_transfer_tasks(
src_cv.cloudpath,
destpath,
chunk_size=(64, 64, 64),
translate=(-128, -128, -64),
)
tq.insert_all(tasks)
dest_cv.refresh_info()
assert len(dest_cv.scales) == 3
assert np.all(src_cv[128:128 + 256, 128:128 + 256,
64:64 + 64] == dest_cv[:])
rmsrc()
rmdest()
def xfer(
ctx, src, dest, queue, translate, downsample, mip,
fill_missing, num_mips, cseg, shape, sparse,
chunk_size, compress, volumetric,
delete_bg, bg_color
):
"""
Transfer an image layer to another location.
It is crucial to choose a good task shape. The task
shape must be a multiple of two of the destination
image layer chunk size. Too small, and you'll have
an inefficient transfer. Too big, and you'll run out
of memory and also have an inefficient transfer.
Downsamples will by default be automatically calculated
from whatever material is available. For the default
2x2x1 downsampling, larger XY dimension is desirable
compared to Z as more downsamples can be computed for
each 2x2 increase in the task size.
"""
encoding = ("compressed_segmentation" if cseg else None)
factor = (2,2,1)
if volumetric:
factor = (2,2,2)
shape = [ int(axis) for axis in shape.split(",") ]
translate = [ int(amt) for amt in translate.split(",") ]
tasks = tc.create_transfer_tasks(
src, dest,
chunk_size=chunk_size, fill_missing=fill_missing,
translate=translate, mip=mip, shape=shape,
encoding=encoding, skip_downsamples=(not downsample),
delete_black_uploads=delete_bg, background_color=bg_color,
compress=compress, factor=factor, sparse=sparse,
)
parallel = int(ctx.obj.get("parallel", 1))
tq = TaskQueue(normalize_path(queue))
tq.insert(tasks, parallel=parallel)
to_upload.sort()
print(f"Have {len(to_upload)} planes to upload")
with ProcessPoolExecutor(max_workers=cpus) as executor:
for job in executor.map(process_slice, to_upload):
try:
print(job)
except Exception as exc:
print(f'generated an exception: {exc}')
elif step == 'step2': # transfer tasks
orig_vol = CloudVolume(f'file://{orig_layer_dir}')
first_chunk = calculate_chunks(downsample='full', mip=0)
tq = LocalTaskQueue(parallel=cpus)
tasks = tc.create_transfer_tasks(orig_vol.cloudpath,
dest_layer_path=rechunked_cloudpath,
chunk_size=first_chunk,
mip=0,
skip_downsamples=True)
print(len(tasks))
tq.insert(tasks)
tq.execute()
elif step == 'step3': # downsampling
print("step 3, downsampling")
tq = LocalTaskQueue(parallel=cpus)
downsample = "full"
mips = [0, 1, 2, 3, 4]
for mip in mips:
print(f"Mip: {mip}")
cv = CloudVolume(rechunked_cloudpath, mip)
chunks = calculate_chunks(downsample, mip)
def create_mesh(animal, limit, mse):
#chunks = calculate_chunks('full', -1)
chunks = [64,64,1]
scales = (10400, 10400, 20000)
fileLocationManager = FileLocationManager(animal)
INPUT = "/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/DK52/preps/CH3/shapes"
OUTPUT1_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'mesh_input')
OUTPUT2_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'mesh')
if 'ultraman' in get_hostname():
if os.path.exists(OUTPUT1_DIR):
shutil.rmtree(OUTPUT1_DIR)
if os.path.exists(OUTPUT2_DIR):
shutil.rmtree(OUTPUT2_DIR)
files = sorted(os.listdir(INPUT))
os.makedirs(OUTPUT1_DIR, exist_ok=True)
os.makedirs(OUTPUT2_DIR, exist_ok=True)
len_files = len(files)
midpoint = len_files // 2
midfilepath = os.path.join(INPUT, files[midpoint])
midfile = io.imread(midfilepath)
data_type = midfile.dtype
#image = np.load('/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/structures/allen/allen.npy')
#ids = np.unique(image)
ids = {'infrahypoglossal': 200, 'perifacial': 210, 'suprahypoglossal': 220}
height, width = midfile.shape
volume_size = (width, height, len(files)) # neuroglancer is width, height
print('volume size', volume_size)
ng = NumpyToNeuroglancer(animal, None, scales, layer_type='segmentation',
data_type=data_type, chunk_size=chunks)
ng.init_precomputed(OUTPUT1_DIR, volume_size, progress_id=1)
file_keys = []
for i,f in enumerate(tqdm(files)):
infile = os.path.join(INPUT, f)
file_keys.append([i, infile])
#ng.process_image([i, infile])
#sys.exit()
start = timer()
workers, cpus = get_cpus()
print(f'Working on {len(file_keys)} files with {workers} cpus')
with ProcessPoolExecutor(max_workers=workers) as executor:
executor.map(ng.process_image, sorted(file_keys), chunksize=1)
executor.shutdown(wait=True)
ng.precomputed_vol.cache.flush()
end = timer()
print(f'Create volume method took {end - start} seconds')
##### rechunk
cloudpath1 = f"file://{OUTPUT1_DIR}"
cv1 = CloudVolume(cloudpath1, 0)
_, workers = get_cpus()
tq = LocalTaskQueue(parallel=workers)
cloudpath2 = f'file://{OUTPUT2_DIR}'
tasks = tc.create_transfer_tasks(cloudpath1, dest_layer_path=cloudpath2,
chunk_size=[64,64,64], mip=0, skip_downsamples=True)
tq.insert(tasks)
tq.execute()
##### add segment properties
cv2 = CloudVolume(cloudpath2, 0)
cv2.info['segment_properties'] = 'names'
cv2.commit_info()
segment_properties_path = os.path.join(cloudpath2.replace('file://', ''), 'names')
os.makedirs(segment_properties_path, exist_ok=True)
info = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids": [str(value) for key, value in ids.items()],
"properties": [{
"id": "label",
"type": "label",
"values": [str(key) for key, value in ids.items()]
}]
}
}
with open(os.path.join(segment_properties_path, 'info'), 'w') as file:
json.dump(info, file, indent=2)
##### first mesh task, create meshing tasks
workers, _ = get_cpus()
tq = LocalTaskQueue(parallel=workers)
mesh_dir = f'mesh_mip_0_err_{mse}'
cv2.info['mesh'] = mesh_dir
cv2.commit_info()
tasks = tc.create_meshing_tasks(cv2.layer_cloudpath, mip=0, mesh_dir=mesh_dir, max_simplification_error=mse)
tq.insert(tasks)
tq.execute()
##### 2nd mesh task, create manifest
tasks = tc.create_mesh_manifest_tasks(cv2.layer_cloudpath, mesh_dir=mesh_dir)
tq.insert(tasks)
tq.execute()
print("Done!")
def xfer(ctx, src, dest, queue, translate, downsample, mip, fill_missing,
memory, max_mips, shape, sparse, cseg, compresso, chunk_size,
compress, volumetric, delete_bg, bg_color, sharded, dest_voxel_offset,
clean_info, no_src_update):
"""
Transfer an image layer to another location.
It is crucial to choose a good task shape. The task
shape must be a multiple of two of the destination
image layer chunk size. Too small, and you'll have
an inefficient transfer. Too big, and you'll run out
of memory and also have an inefficient transfer.
Downsamples will by default be automatically calculated
from whatever material is available. For the default
2x2x1 downsampling, larger XY dimension is desirable
compared to Z as more downsamples can be computed for
each 2x2 increase in the task size.
Use the --memory flag to automatically compute the
a reasonable task shape based on your memory limits.
"""
src = cloudfiles.paths.normalize(src)
dest = cloudfiles.paths.normalize(dest)
if cseg and compresso:
print("igneous: must choose one of --cseg or --compresso")
sys.exit()
encoding = None
if cseg:
encoding = "compressed_segmentation"
elif compresso:
encoding = "compresso"
factor = (2, 2, 1)
if volumetric:
factor = (2, 2, 2)
if compress and compress.lower() == "false":
compress = False
if sharded:
tasks = tc.create_image_shard_transfer_tasks(
src,
dest,
chunk_size=chunk_size,
fill_missing=fill_missing,
mip=mip,
dest_voxel_offset=dest_voxel_offset,
translate=translate,
encoding=encoding,
memory_target=memory,
clean_info=clean_info)
else:
tasks = tc.create_transfer_tasks(src,
dest,
chunk_size=chunk_size,
fill_missing=fill_missing,
dest_voxel_offset=dest_voxel_offset,
translate=translate,
mip=mip,
shape=shape,
encoding=encoding,
skip_downsamples=(not downsample),
delete_black_uploads=delete_bg,
background_color=bg_color,
compress=compress,
factor=factor,
sparse=sparse,
memory_target=memory,
max_mips=max_mips,
clean_info=clean_info,
no_src_update=no_src_update)
parallel = int(ctx.obj.get("parallel", 1))
tq = TaskQueue(normalize_path(queue))
tq.insert(tasks, parallel=parallel)