def test_get_img_info_uint16_tif_s3(self):
self.set_s3_args()
ingest_job = IngestJob(self.args)
z_slice = 0
# create an image
img_fname = ingest_job.get_img_fname(z_slice)
create_img_file(ingest_job.img_size[0], ingest_job.img_size[1],
self.args.datatype, self.args.extension, img_fname)
# put the image on a bucket
s3 = boto3.resource('s3')
with open(img_fname, 'rb') as img_file:
s3.Bucket(self.args.s3_bucket_name).put_object(Key=img_fname,
Body=img_file)
# get info on that image
im_width, im_height, im_datatype = ingest_job.get_img_info(z_slice)
# assert the info is correct
im = np.array(Image.open(img_fname))
assert im_width == im.shape[1]
assert im_height == im.shape[0]
assert im_datatype == im.dtype
s3.Bucket(self.args.s3_bucket_name).delete_objects(
Delete={'Objects': [{
'Key': img_fname
}]})
os.remove(ingest_job.get_log_fname())
os.remove(img_fname)
def test_get_boss_resources(self):
coll, exp, ch, x, y, z = parse_cut_line(self.cutout_text)
datasource, _, aws_profile, boss_config_file, base_path, base_filename, extension, z_step, datatype = create_local_ingest_params(
)
args = Namespace(
datasource=datasource,
collection=coll,
experiment=exp,
channel=ch,
datatype=datatype,
aws_profile=aws_profile,
boss_config_file=boss_config_file,
base_path=base_path,
base_filename=base_filename,
extension=extension,
z_range=[0, 16],
z_step=z_step,
warn_missing_files=True,
get_extents=True,
res=0,
)
ingest_job = IngestJob(args)
boss_res_params = BossResParams(ingest_job)
boss_res_params.setup_boss_coord_frame(get_only=True)
boss_res_params.get_resources(get_only=False)
assert boss_res_params.coll_resource.name == coll
assert boss_res_params.exp_resource.name == exp
assert boss_res_params.ch_resource.name == ch
os.remove(ingest_job.get_log_fname())
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
def test_ingest_uint8_annotations(self):
dtype = 'uint8'
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
self.args.base_filename = 'img_annotation_<p:4>'
self.args.channel = 'def_files_annotation_' + now
self.args.channels_list_file = None
self.args.source_channel = 'def_files'
self.args.datatype = dtype
self.args.extension = 'tif'
self.args.create_resources = True
ingest_job = IngestJob(self.args)
gen_images(ingest_job, intensity_range=30)
channel = self.args.channel
result = per_channel_ingest(self.args, channel)
assert result == 0
self.args.create_resources = False
result = per_channel_ingest(self.args, channel)
assert result == 0
# cleanup
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
del_test_images(ingest_job)
os.remove(ingest_job.get_log_fname())
def test_post_uint16_cutout(self):
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
x_size = 128
y_size = 128
dtype = 'uint16'
bit_width = int(''.join(filter(str.isdigit, dtype)))
# generate a block of data
data = np.random.randint(
1, 2**bit_width, size=(self.args.z_range[1], y_size, x_size), dtype=dtype)
# post (non-zero) data to boss
st_x, sp_x, st_y, sp_y, st_z, sp_z = (
0, x_size, 0, y_size, 0, self.args.z_range[1])
self.args.datatype = dtype
self.args.channel = 'def_files' + now
ingest_job = IngestJob(self.args)
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=False)
ret_val = post_cutout(boss_res_params, ingest_job, [st_x, sp_x], [st_y, sp_y],
[st_z, sp_z], data, attempts=1)
assert ret_val == 0
# read data out of boss
data_boss = boss_res_params.rmt.get_cutout(boss_res_params.ch_resource, 0,
[st_x, sp_x], [st_y, sp_y], [st_z, sp_z])
# assert they are the same
assert np.array_equal(data_boss, data)
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
os.remove(ingest_job.get_log_fname())
def test_get_boss_annotation_channel(self):
datatype = 'uint64'
args = Namespace(datasource='local',
collection='ben_dev',
experiment='dev_ingest_4',
channel='def_files_annot',
boss_config_file=None,
source_channel='def_files',
x_extent=[0, 1000],
y_extent=[0, 1024],
z_extent=[0, 100])
ingest_job = IngestJob(args)
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
assert ingest_job.ch_name == boss_res_params.ch_resource.name
assert ingest_job.boss_datatype == datatype
assert ingest_job.ch_type == 'annotation'
assert boss_res_params.ch_resource.type == 'annotation'
assert boss_res_params.ch_resource.sources == [args.source_channel]
os.remove(ingest_job.get_log_fname())
def test_create_local_IngestJob(self):
ingest_job = IngestJob(self.args)
assert ingest_job.x_extent == self.args.x_extent
assert ingest_job.base_fname == self.args.base_filename
os.remove(ingest_job.get_log_fname())
def test_get_boss_res_wrong_img_size(self):
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
x_extent = [0, 2000]
y_extent = [0, 1000]
z_extent = [0, 50]
voxel_size = [1, 5, 1]
args = Namespace(datasource='local',
collection='ben_dev',
experiment='dev_ingest_4' + now,
channel='def_files_' + now,
boss_config_file=None,
voxel_size=voxel_size,
voxel_unit='nanometers',
datatype='uint16',
res=0,
x_extent=x_extent,
y_extent=y_extent,
z_extent=z_extent)
ingest_job = IngestJob(args)
with pytest.raises(HTTPError):
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
os.remove(ingest_job.get_log_fname())
def test_per_channel_ingest_neg_z_extent_offset(self):
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
self.args.experiment = 'test_neg_offset_' + now
self.args.channel = 'def_files'
self.args.datatype = 'uint8'
self.args.z_extent = [-100, 100]
self.args.z_range = [-3, 2]
self.args.offset_extents = True
self.args.extension = 'png'
ingest_job = IngestJob(self.args)
gen_images(ingest_job)
self.args.create_resources = True
result = per_channel_ingest(self.args, self.args.channel)
assert result == 0
self.args.create_resources = False
result = per_channel_ingest(self.args, self.args.channel)
assert result == 0
# cleanup
del_test_images(ingest_job)
os.remove(ingest_job.get_log_fname())
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
boss_res_params.rmt.delete_project(boss_res_params.exp_resource)
def test_per_channel_ingest_neg_x_extent_offset(self):
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
self.args.experiment = 'test_neg_offset_' + now
self.args.channel = 'def_files'
self.args.datatype = 'uint16'
self.args.x_extent = [-1000, 0]
self.args.offset_extents = True
self.args.extension = 'png'
self.args.channels_list_file = 'tests/channels.test.txt'
channels = read_channel_names(self.args.channels_list_file)
# assertions are inside ingest_test_per_channel
# this is to create resources only:
self.args.create_resources = True
self.ingest_test_per_channel(self.args, channels)
self.args.create_resources = False
self.ingest_test_per_channel(self.args, channels)
# cleanup
for ch in channels:
ch_args = self.args
ch_args.channel = ch
ingest_job = IngestJob(ch_args)
os.remove(ingest_job.get_log_fname())
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
if len(channels) > 0:
boss_res_params.rmt.delete_project(boss_res_params.exp_resource)
def test_create_boss_res_specified_coord_frame(self):
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
args = Namespace(datasource='local',
collection='ben_dev',
experiment='dev_ingest_neg' + now,
channel='def_files_' + now,
boss_config_file=None,
voxel_size=[1, 5, 1],
voxel_unit='nanometers',
datatype='uint16',
res=0,
x_extent=[100, 1100],
y_extent=[0, 1024],
z_extent=[200, 300],
coord_frame_x_extent=[0, 2000])
ingest_job = IngestJob(args)
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=False)
assert boss_res_params.coord_frame_resource.z_start == 200
assert boss_res_params.coord_frame_resource.z_stop == 300
assert boss_res_params.coord_frame_resource.x_start == 0
assert boss_res_params.coord_frame_resource.x_stop == 2000
assert ingest_job.x_extent == [100, 1100]
assert ingest_job.y_extent == [0, 1024]
assert ingest_job.offsets == [0, 0, 0]
os.remove(ingest_job.get_log_fname())
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
boss_res_params.rmt.delete_project(boss_res_params.exp_resource)
def test_get_boss_res_params_just_names(self):
args = Namespace(datasource='local',
collection='ben_dev',
experiment='dev_ingest_4',
channel='def_files',
boss_config_file=None,
x_extent=[0, 1000],
y_extent=[0, 1024],
z_extent=[0, 100])
ingest_job = IngestJob(args)
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
assert boss_res_params.coll_resource.name == args.collection
assert boss_res_params.exp_resource.name == args.experiment
assert boss_res_params.ch_resource.name == args.channel
assert boss_res_params.exp_resource.hierarchy_method == 'isotropic'
assert ingest_job.voxel_size == [1, 1, 1]
assert ingest_job.voxel_unit == 'micrometers'
assert ingest_job.offsets == [0, 0, 0]
assert ingest_job.datatype == 'uint16'
assert ingest_job.boss_datatype == 'uint16'
assert ingest_job.res == 0
assert ingest_job.extension is None
assert ingest_job.z_step is None
os.remove(ingest_job.get_log_fname())
def test_create_local_IngestJob_specific_coord_frame(self):
self.args.coord_frame_x_extent = [0, 2000]
ingest_job = IngestJob(self.args)
assert ingest_job.coord_frame_x_extent == [0, 2000]
assert ingest_job.coord_frame_y_extent == [0, 1024]
os.remove(ingest_job.get_log_fname())
def test_get_img_fname(self):
self.args.extension = 'tif'
ingest_job = IngestJob(self.args)
img_fname = ingest_job.get_img_fname(0)
img_fname_test = '{}img_{:04d}.{}'.format(self.args.base_path, 0,
self.args.extension)
assert img_fname == img_fname_test
os.remove(ingest_job.get_log_fname())
def test_get_img_fname_channel(self):
self.args.base_filename = 'img_<ch>_<p:4>'
self.args.base_path = 'local_img_<ch>_test_data/'
ingest_job = IngestJob(self.args)
img_fname = ingest_job.get_img_fname(0)
assert img_fname == 'local_img_{0}_test_data/img_{0}_{1:04d}.tif'.format(
self.args.channel, 0)
os.remove(ingest_job.get_log_fname())
def test_create_local_IngestJob_pos_extents_offset(self):
self.args.x_extent = [1000, 2000]
self.args.offset_extents = True
ingest_job = IngestJob(self.args)
assert ingest_job.offsets == [0, 0, 0]
assert ingest_job.x_extent == [1000, 2000]
os.remove(ingest_job.get_log_fname())
def test_create_local_IngestJob_annotation_uint32(self):
self.args.source_channel = 'def_files'
self.args.datatype = 'uint32'
ingest_job = IngestJob(self.args)
assert ingest_job.source_channel == self.args.source_channel
assert ingest_job.boss_datatype == 'uint64'
assert ingest_job.datatype == 'uint32'
os.remove(ingest_job.get_log_fname())
def test_send_msg_slack(self):
self.args.slack_usr = '******'
ingest_job = IngestJob(self.args)
assert ingest_job.slack_obj is not None
msg = 'test_message_' + datetime.now().strftime("%Y-%m-%d %H:%M:%S")
ingest_job.send_msg(msg, send_slack=True)
log_fname = ingest_job.get_log_fname()
os.remove(log_fname)
def test_create_local_IngestJob_neg_extents_forced_offsets(self):
self.args.x_extent = [-1000, -100]
self.args.forced_offsets = [1100, 100, 200]
ingest_job = IngestJob(self.args)
assert ingest_job.offsets == [1100, 100, 200]
assert ingest_job.x_extent == [100, 1000]
assert ingest_job.y_extent == [100, 1124]
assert ingest_job.z_extent == [200, 300]
os.remove(ingest_job.get_log_fname())
def test_create_local_IngestJob_only_coord_extents(self):
self.args.coord_frame_x_extent = [0, 1000]
self.args.coord_frame_y_extent = [0, 1024]
self.args.coord_frame_z_extent = [0, 100]
ingest_job = IngestJob(self.args)
assert ingest_job.x_extent == [0, 1000]
assert ingest_job.coord_frame_x_extent == [0, 1000]
assert ingest_job.y_extent == [0, 1024]
assert ingest_job.coord_frame_y_extent == [0, 1024]
os.remove(ingest_job.get_log_fname())
def test_send_msg(self):
ingest_job = IngestJob(self.args)
msg = 'test_message_' + datetime.now().strftime("%Y-%m-%d %H:%M:%S")
ingest_job.send_msg(msg)
log_fname = ingest_job.get_log_fname()
with open(log_fname) as f:
log_data = f.readlines()
assert msg in log_data[-1]
os.remove(log_fname)
def test_local_ingest_cuts(self):
cut = create_cutout(self.cutout_text)
coll, exp, ch = (cut.collection, cut.experiment, cut.channel)
datasource, s3_bucket_name, aws_profile, boss_config_file, base_path, base_filename, extension, z_step, datatype = create_local_ingest_params(
)
args = Namespace(
datasource=datasource,
s3_bucket_name=s3_bucket_name,
collection=coll,
experiment=exp,
channel=ch,
datatype=datatype,
aws_profile=aws_profile,
boss_config_file=boss_config_file,
base_path=base_path,
base_filename=base_filename,
extension=extension,
z_range=[0, 16],
z_step=z_step,
warn_missing_files=True,
get_extents=True,
res=0,
)
ingest_job = IngestJob(args)
boss_res_params = BossResParams(ingest_job)
boss_res_params.setup_boss_coord_frame(get_only=True)
boss_res_params.get_resources(get_only=False)
gen_images(ingest_job)
# ingest the cut
ingest_cuts([cut], ingest_job, boss_res_params)
# pull the data from the boss after the new ingest
data_boss = boss_res_params.rmt.get_cutout(boss_res_params.ch_resource,
0, cut.x, cut.y, cut.z)
# test to make sure it's the same as local file
z_slices = range(cut.z[0], cut.z[1])
# loading data locally for comparison
im_array = ingest_job.read_img_stack(z_slices)
data_local = im_array[:, cut.y[0]:cut.y[1], cut.x[0]:cut.x[1]]
assert np.array_equal(data_local, data_boss)
del_test_images(ingest_job)
os.remove(ingest_job.get_log_fname())
os.remove(cut.log_fname)
def iterate_posting_cutouts(cutouts):
# separate the cutouts into groupings of shared collections/experiments/channels
collections = set([cu.collection for cu in cutouts])
for coll in collections:
cus_coll = [cu for cu in cutouts if cu.collection == coll]
experiments = set([cu.experiment for cu in cus_coll])
for exp in experiments:
cus_exp = [cu for cu in cus_coll if cu.experiment == exp]
channels = set([cu.channel for cu in cus_exp])
for ch in channels:
cus_ch = [cu for cu in cus_exp if cu.channel == ch]
if len(cus_ch) > 0:
# posts data for cutouts that share a common coll, exp, and ch
msg = 'Repeating cutouts for collection {}, experiment {}, channel {}'.format(
coll, exp, ch)
cus_ch[-1].send_msg(msg)
args = gather_info()
args.collection = coll
args.experiment = exp
args.channel = ch
args.get_extents = True
ingest_job = IngestJob(args)
# we get these things from the resources that already exist on the boss:
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=True)
ingest_cuts(cus_ch, ingest_job, boss_res_params)
def test_create_local_IngestJob_no_extents(self):
self.args.x_extent = None
self.args.y_extent = None
self.args.z_extent = None
with pytest.raises(ValueError):
IngestJob(self.args)
def test_load_img_local(self):
ingest_job = IngestJob(self.args)
z_slice = 0
img_fname = ingest_job.get_img_fname(z_slice)
create_img_file(ingest_job.img_size[0], ingest_job.img_size[1],
self.args.datatype, self.args.extension, img_fname)
im = ingest_job.load_img(z_slice)
img_local_test = np.array(Image.open(img_fname))
assert np.array_equal(im, img_local_test)
os.remove(ingest_job.get_log_fname())
os.remove(img_fname)
def ingest_test_per_channel(self, args, channels):
for channel in channels:
args.channel = channel
ingest_job = IngestJob(args)
gen_images(ingest_job)
result = per_channel_ingest(args, channel)
assert result == 0
del_test_images(ingest_job)
def test_get_img_info_uint16_png(self):
file_format = 'png'
self.args.file_format = file_format
ingest_job = IngestJob(self.args)
z_slice = 0
img_fname = ingest_job.get_img_fname(z_slice)
create_img_file(ingest_job.img_size[0], ingest_job.img_size[1],
self.args.datatype, self.args.extension, img_fname)
im_width, im_height, im_datatype = ingest_job.get_img_info(z_slice)
assert im_width == ingest_job.img_size[0]
assert im_height == ingest_job.img_size[1]
assert im_datatype == self.args.datatype
os.remove(ingest_job.get_log_fname())
os.remove(img_fname)
def test_read_uint16_img_stack(self):
ingest_job = IngestJob(self.args)
# generate some images
gen_images(ingest_job)
# load images into memory using ingest_job
z_slices = range(self.args.z_range[0], self.args.z_range[1])
im_array = ingest_job.read_img_stack(z_slices)
# check to make sure each image is equal to each z index in the array
for z in z_slices:
img_fname = self.args.base_path + 'img_{:04d}.tif'.format(z)
with Image.open(img_fname) as im:
assert np.array_equal(im_array[z, :, :], im)
del_test_images(ingest_job)
os.remove(ingest_job.get_log_fname())
def test_create_local_IngestJob_neg_z_extents_offset_range(self):
self.args.z_extent = [-1000, 2000]
self.args.z_range = [-10, 10]
self.args.offset_extents = True
ingest_job = IngestJob(self.args)
assert ingest_job.x_extent == [0, 1000]
assert ingest_job.offsets == [0, 0, 1000]
assert ingest_job.z_extent == [0, 3000]
assert ingest_job.z_range == [-10, 10]
img_fname = ingest_job.get_img_fname(-5)
# assert that first tif image has a file name with the negative original extent of the data
img_fname_test = '{}img_{:04d}.{}'.format(self.args.base_path, -5,
self.args.extension)
assert img_fname == img_fname_test
os.remove(ingest_job.get_log_fname())
def test_get_img_info_uint64_tif(self):
file_format = 'tif'
dtype = 'uint64'
self.args.file_format = file_format
self.args.datatype = dtype
self.args.source_channel = 'def_files'
ingest_job = IngestJob(self.args)
z_slice = 0
img_fname = ingest_job.get_img_fname(z_slice)
create_img_file(ingest_job.img_size[0], ingest_job.img_size[1],
self.args.datatype, self.args.extension, img_fname)
im_width, im_height, im_datatype = ingest_job.get_img_info(z_slice)
assert im_width == ingest_job.img_size[0]
assert im_height == ingest_job.img_size[1]
assert im_datatype == self.args.datatype
os.remove(ingest_job.get_log_fname())
os.remove(img_fname)
def test_create_boss_res_offsets(self):
now = datetime.now().strftime("%Y%m%d-%H%M%S%f")
args = Namespace(datasource='local',
collection='ben_dev',
experiment='dev_ingest_neg' + now,
channel='def_files_' + now,
boss_config_file=None,
voxel_size=[1, 5, 1],
voxel_unit='nanometers',
datatype='uint16',
res=0,
x_extent=[-500, 500],
y_extent=[0, 1024],
z_extent=[200, 300],
offset_extents=True)
ingest_job = IngestJob(args)
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=False)
assert boss_res_params.coord_frame_resource.z_start == 200
assert boss_res_params.coord_frame_resource.z_stop == 300
assert boss_res_params.coord_frame_resource.x_start == 0
assert boss_res_params.coord_frame_resource.x_stop == 1000
assert ingest_job.offsets == [500, 0, 0]
# testing to make sure offsets were recorded properly
exp_res = boss_res_params.exp_resource
boss_offsets_dict = boss_res_params.rmt.get_metadata(
exp_res, ['offsets'])
boss_offsets = ast.literal_eval(boss_offsets_dict['offsets'])
assert boss_offsets == [500, 0, 0]
os.remove(ingest_job.get_log_fname())
boss_res_params.rmt.delete_project(boss_res_params.ch_resource)
boss_res_params.rmt.delete_project(boss_res_params.exp_resource)