def test_per_channel_ingest_neg_z_extent_offset(self):
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
self.args.experiment = 'test_neg_offset_' + now
self.args.channel = 'def_files'
self.args.datatype = 'uint8'
self.args.z_extent = [-100, 100]
self.args.z_range = [-3, 2]
self.args.offset_extents = True
self.args.extension = 'png'
ingest_job = IngestJob(self.args)
gen_images(ingest_job)
self.args.create_resources = True
result = per_channel_ingest(self.args, self.args.channel)
assert result == 0
self.args.create_resources = False
result = per_channel_ingest(self.args, self.args.channel)
assert result == 0
# cleanup
del_test_images(ingest_job)
os.remove(ingest_job.get_log_fname())
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
boss_res_params.rmt.delete_project(boss_res_params.exp_resource)
def test_ingest_uint8_annotations(self):
dtype = 'uint8'
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
self.args.base_filename = 'img_annotation_<p:4>'
self.args.channel = 'def_files_annotation_' + now
self.args.channels_list_file = None
self.args.source_channel = 'def_files'
self.args.datatype = dtype
self.args.extension = 'tif'
self.args.create_resources = True
ingest_job = IngestJob(self.args)
gen_images(ingest_job, intensity_range=30)
channel = self.args.channel
result = per_channel_ingest(self.args, channel)
assert result == 0
self.args.create_resources = False
result = per_channel_ingest(self.args, channel)
assert result == 0
# cleanup
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
del_test_images(ingest_job)
os.remove(ingest_job.get_log_fname())
def ingest_test_per_channel(self, args, channels):
for channel in channels:
args.channel = channel
ingest_job = IngestJob(args)
gen_images(ingest_job)
result = per_channel_ingest(args, channel)
assert result == 0
del_test_images(ingest_job)
def test_ingest_blocks_uint16_8_threads(self):
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
self.args.experiment = 'dev_ingest_larger' + now
self.args.channel = 'def_files' + now
self.args.x_extent = [0, 8 * 1024]
self.args.z_range = [0, 16]
self.args.datatype = 'uint16'
self.args.extension = 'tif'
x_size = 8 * 1024
y_size = 1024
stride_x = 1024
x_buckets = get_supercube_lims(self.args.x_extent, stride_x)
ingest_job = IngestJob(self.args)
gen_images(ingest_job)
self.args.create_resources = True
result = per_channel_ingest(self.args, self.args.channel)
assert result == 0
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
z_slices = list(range(self.args.z_range[0], self.args.z_range[-1]))
y_rng = self.args.y_extent
im_array = ingest_job.read_img_stack(z_slices)
threads = 8
ingest_block_partial = partial(
ingest_block, x_buckets=x_buckets, boss_res_params=boss_res_params, ingest_job=ingest_job,
y_rng=y_rng, z_rng=self.args.z_range, im_array=im_array)
start_time = time.time()
with ThreadPool(threads) as pool:
pool.map(ingest_block_partial, x_buckets.keys())
time_taken = time.time() - start_time
print('{} secs taken with {} threads'.format(time_taken, threads))
data_boss = download_boss_slice(
boss_res_params, ingest_job, 0)[0, :, :]
data_local = im_array[0, :, :]
assert np.array_equal(data_boss, data_local)
# cleanup
ingest_job = IngestJob(self.args)
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
boss_res_params.rmt.delete_project(boss_res_params.exp_resource)
os.remove(ingest_job.get_log_fname())
del_test_images(ingest_job)
def test_per_channel_ingest_wrong_datatype(self):
# create 16 bit images and post to 8 bit resource
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
self.args.channel = 'def_files_8bit_' + now
self.args.extension = 'tif'
# make 16 bit images
args_uint16 = self.args
args_uint16.datatype = 'uint16'
ingest_job_uint16 = IngestJob(args_uint16)
gen_images(ingest_job_uint16)
# try to do an 8 bit ingest on a 16 bit resource with 16 bit files
self.args.datatype = 'uint8'
self.args.create_resources = True # create the resource
with pytest.raises(ValueError):
per_channel_ingest(self.args, self.args.channel)
# cleanup
del_test_images(ingest_job_uint16)
ingest_job = IngestJob(self.args)
os.remove(ingest_job.get_log_fname())