def test_per_channel_ingest_neg_z_extent_offset(self): now = datetime.now().strftime("%Y%m%d-%H%M%S%f") self.args.experiment = 'test_neg_offset_' + now self.args.channel = 'def_files' self.args.datatype = 'uint8' self.args.z_extent = [-100, 100] self.args.z_range = [-3, 2] self.args.offset_extents = True self.args.extension = 'png' ingest_job = IngestJob(self.args) gen_images(ingest_job) self.args.create_resources = True result = per_channel_ingest(self.args, self.args.channel) assert result == 0 self.args.create_resources = False result = per_channel_ingest(self.args, self.args.channel) assert result == 0 # cleanup del_test_images(ingest_job) os.remove(ingest_job.get_log_fname()) boss_res_params = BossResParams(ingest_job) boss_res_params.get_resources(get_only=True) boss_res_params.rmt.delete_project(boss_res_params.ch_resource) boss_res_params.rmt.delete_project(boss_res_params.exp_resource)
def test_ingest_uint8_annotations(self): dtype = 'uint8' now = datetime.now().strftime("%Y%m%d-%H%M%S%f") self.args.base_filename = 'img_annotation_<p:4>' self.args.channel = 'def_files_annotation_' + now self.args.channels_list_file = None self.args.source_channel = 'def_files' self.args.datatype = dtype self.args.extension = 'tif' self.args.create_resources = True ingest_job = IngestJob(self.args) gen_images(ingest_job, intensity_range=30) channel = self.args.channel result = per_channel_ingest(self.args, channel) assert result == 0 self.args.create_resources = False result = per_channel_ingest(self.args, channel) assert result == 0 # cleanup boss_res_params = BossResParams(ingest_job) boss_res_params.get_resources(get_only=True) boss_res_params.rmt.delete_project(boss_res_params.ch_resource) del_test_images(ingest_job) os.remove(ingest_job.get_log_fname())
def ingest_test_per_channel(self, args, channels): for channel in channels: args.channel = channel ingest_job = IngestJob(args) gen_images(ingest_job) result = per_channel_ingest(args, channel) assert result == 0 del_test_images(ingest_job)
def test_ingest_blocks_uint16_8_threads(self): now = datetime.now().strftime("%Y%m%d-%H%M%S%f") self.args.experiment = 'dev_ingest_larger' + now self.args.channel = 'def_files' + now self.args.x_extent = [0, 8 * 1024] self.args.z_range = [0, 16] self.args.datatype = 'uint16' self.args.extension = 'tif' x_size = 8 * 1024 y_size = 1024 stride_x = 1024 x_buckets = get_supercube_lims(self.args.x_extent, stride_x) ingest_job = IngestJob(self.args) gen_images(ingest_job) self.args.create_resources = True result = per_channel_ingest(self.args, self.args.channel) assert result == 0 boss_res_params = BossResParams(ingest_job) boss_res_params.get_resources(get_only=True) z_slices = list(range(self.args.z_range[0], self.args.z_range[-1])) y_rng = self.args.y_extent im_array = ingest_job.read_img_stack(z_slices) threads = 8 ingest_block_partial = partial( ingest_block, x_buckets=x_buckets, boss_res_params=boss_res_params, ingest_job=ingest_job, y_rng=y_rng, z_rng=self.args.z_range, im_array=im_array) start_time = time.time() with ThreadPool(threads) as pool: pool.map(ingest_block_partial, x_buckets.keys()) time_taken = time.time() - start_time print('{} secs taken with {} threads'.format(time_taken, threads)) data_boss = download_boss_slice( boss_res_params, ingest_job, 0)[0, :, :] data_local = im_array[0, :, :] assert np.array_equal(data_boss, data_local) # cleanup ingest_job = IngestJob(self.args) boss_res_params = BossResParams(ingest_job) boss_res_params.get_resources(get_only=True) boss_res_params.rmt.delete_project(boss_res_params.ch_resource) boss_res_params.rmt.delete_project(boss_res_params.exp_resource) os.remove(ingest_job.get_log_fname()) del_test_images(ingest_job)
def test_local_ingest_cuts(self): cut = create_cutout(self.cutout_text) coll, exp, ch = (cut.collection, cut.experiment, cut.channel) datasource, s3_bucket_name, aws_profile, boss_config_file, base_path, base_filename, extension, z_step, datatype = create_local_ingest_params( ) args = Namespace( datasource=datasource, s3_bucket_name=s3_bucket_name, collection=coll, experiment=exp, channel=ch, datatype=datatype, aws_profile=aws_profile, boss_config_file=boss_config_file, base_path=base_path, base_filename=base_filename, extension=extension, z_range=[0, 16], z_step=z_step, warn_missing_files=True, get_extents=True, res=0, ) ingest_job = IngestJob(args) boss_res_params = BossResParams(ingest_job) boss_res_params.setup_boss_coord_frame(get_only=True) boss_res_params.get_resources(get_only=False) gen_images(ingest_job) # ingest the cut ingest_cuts([cut], ingest_job, boss_res_params) # pull the data from the boss after the new ingest data_boss = boss_res_params.rmt.get_cutout(boss_res_params.ch_resource, 0, cut.x, cut.y, cut.z) # test to make sure it's the same as local file z_slices = range(cut.z[0], cut.z[1]) # loading data locally for comparison im_array = ingest_job.read_img_stack(z_slices) data_local = im_array[:, cut.y[0]:cut.y[1], cut.x[0]:cut.x[1]] assert np.array_equal(data_local, data_boss) del_test_images(ingest_job) os.remove(ingest_job.get_log_fname()) os.remove(cut.log_fname)
def test_read_uint16_img_stack(self): ingest_job = IngestJob(self.args) # generate some images gen_images(ingest_job) # load images into memory using ingest_job z_slices = range(self.args.z_range[0], self.args.z_range[1]) im_array = ingest_job.read_img_stack(z_slices) # check to make sure each image is equal to each z index in the array for z in z_slices: img_fname = self.args.base_path + 'img_{:04d}.tif'.format(z) with Image.open(img_fname) as im: assert np.array_equal(im_array[z, :, :], im) del_test_images(ingest_job) os.remove(ingest_job.get_log_fname())
def test_per_channel_ingest_wrong_datatype(self): # create 16 bit images and post to 8 bit resource now = datetime.now().strftime("%Y%m%d-%H%M%S%f") self.args.channel = 'def_files_8bit_' + now self.args.extension = 'tif' # make 16 bit images args_uint16 = self.args args_uint16.datatype = 'uint16' ingest_job_uint16 = IngestJob(args_uint16) gen_images(ingest_job_uint16) # try to do an 8 bit ingest on a 16 bit resource with 16 bit files self.args.datatype = 'uint8' self.args.create_resources = True # create the resource with pytest.raises(ValueError): per_channel_ingest(self.args, self.args.channel) # cleanup del_test_images(ingest_job_uint16) ingest_job = IngestJob(self.args) os.remove(ingest_job.get_log_fname())