def test_init(self):
data_storage = s3_storage.S3Storage(
storage_dir=self.storage_dir,
nbr_workers=self.nbr_workers,
access_point='test_bucket_name',
)
self.assertEqual(data_storage.bucket_name, 'test_bucket_name')
def test_upload_serialized(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
key = "/".join([self.storage_dir, self.im_name])
key_byte_tuple = (key, self.im_encoded)
data_storage.upload_serialized(key_byte_tuple)
byte_string = self.conn.Object(self.bucket_name,
key).get()['Body'].read()
nose.tools.assert_equal(byte_string, self.im_encoded)
def test_upload_file_get_im(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
data_storage.upload_file(file_path=self.file_path)
# Load the temporary image
im_out = data_storage.get_im(file_name=self.im_name)
# Assert that contents are the same
nose.tools.assert_equal(im_out.dtype, np.uint16)
numpy.testing.assert_array_equal(im_out, self.im)
def test_upload_im(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
key = "/".join([self.storage_dir, self.im_name])
data_storage.upload_im(im_name=self.im_name, im=self.im)
byte_string = self.conn.Object(self.bucket_name,
key).get()['Body'].read()
im = im_utils.deserialize_im(byte_string)
numpy.testing.assert_array_equal(im, self.im)
def test_upload_existing_im(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
key = "/".join([self.storage_dir, self.im_name])
data_storage.upload_im(im_name=self.im_name, im=self.im)
with captured_output() as (out, err):
data_storage.upload_im(im_name=self.im_name, im=self.im)
std_output = out.getvalue().strip()
self.assertEqual(
std_output,
"Key {} already exists.".format(key),
)
def test_get_stack(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
data_storage.upload_frames(self.stack_names, self.im_stack)
# Load image stack in memory
im_out = data_storage.get_stack(self.stack_names, )
nose.tools.assert_equal(self.im_stack.shape, im_out.shape)
for im_nbr in range(self.im_stack.shape[-1]):
# Assert that contents are the same
numpy.testing.assert_array_equal(
im_out[..., im_nbr],
self.im_stack[..., im_nbr],
)
def migrate_db(credentials_filename):
"""
Updates sha256 checksums for all files and frames
:param credentials_filename: Full path to DB credentials file
"""
# Edit this depending on where your database credential file is stored
# This assumes it's stored in dir above imagingDB
dir_name = os.path.abspath(os.path.join('..'))
dest_dir = os.path.join(dir_name, 'temp_downloads')
os.makedirs(dest_dir, exist_ok=True)
credentials_str = db_utils.get_connection_str(
credentials_filename=credentials_filename, )
# Get files and compute checksums
with db_ops.session_scope(credentials_str) as session:
files = session.query(db_ops.FileGlobal)
for file in files:
if file.sha256 is None:
data_loader = s3_storage.S3Storage(
storage_dir=file.storage_dir, )
file_name = file.metadata_json["file_origin"]
file_name = file_name.split("/")[-1]
dest_path = os.path.join(dest_dir, file_name)
data_loader.download_file(
file_name=file_name,
dest_path=dest_path,
)
checksum = meta_utils.gen_sha256(dest_path)
file.sha256 = checksum
# Get frames and compute checksums
with db_ops.session_scope(credentials_filename) as session:
frames = session.query(db_ops.Frames)
for frame in frames:
if frame.sha256 is None:
data_loader = s3_storage.S3Storage(
storage_dir=frame.frames_global.storage_dir, )
im = data_loader.get_im(frame.file_name)
checksum = meta_utils.gen_sha256(im)
frame.sha256 = checksum
def test_download_file(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
data_storage.upload_file(file_path=self.file_path)
# Download the temporary image then read it and validate
data_storage.download_file(
file_name=self.im_name,
dest_dir=self.temp_path,
)
# Read downloaded file and assert that contents are the same
dest_path = os.path.join(self.temp_path, self.im_name)
im_out = cv2.imread(dest_path,
cv2.IMREAD_ANYCOLOR | cv2.IMREAD_ANYDEPTH)
nose.tools.assert_equal(im_out.dtype, np.uint16)
numpy.testing.assert_array_equal(im_out, self.im)
def test_upload_file(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
data_storage.upload_file(file_path=self.file_path)
# Make sure the image uploaded in setUp is unchanged
key = "/".join([self.storage_dir, self.im_name])
byte_string = self.conn.Object(
self.bucket_name,
key,
).get()['Body'].read()
# Construct an array from the bytes and decode image
im_out = im_utils.deserialize_im(byte_string)
# Assert that contents are the same
nose.tools.assert_equal(im_out.dtype, np.uint16)
numpy.testing.assert_array_equal(im_out, self.im)
def test_upload_frames(self):
# Upload image stack
storage_dir = "raw_frames/ML-2005-05-23-10-00-00-0001"
data_storage = s3_storage.S3Storage(storage_dir, self.nbr_workers)
data_storage.upload_frames(self.stack_names, self.im_stack)
# Get images from uploaded stack and validate that the contents are unchanged
for im_nbr in range(len(self.stack_names)):
key = "/".join([storage_dir, self.stack_names[im_nbr]])
byte_string = self.conn.Object(self.bucket_name,
key).get()['Body'].read()
# Construct an array from the bytes and decode image
im = im_utils.deserialize_im(byte_string)
# Assert that contents are the same
nose.tools.assert_equal(im.dtype, np.uint16)
nose.tools.assert_equal(im.shape, (10, 15))
numpy.testing.assert_array_equal(im, self.im_stack[..., im_nbr])
def test_get_stack_with_shape(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
data_storage.upload_frames(self.stack_names, self.im_stack)
# Load image stack in memory
stack_shape = (10, 15, 1, 2)
im_out = data_storage.get_stack_with_shape(self.stack_names,
stack_shape=stack_shape,
bit_depth=np.uint16)
im_out = np.squeeze(im_out)
nose.tools.assert_equal(self.im_stack.shape, im_out.shape)
for im_nbr in range(self.im_stack.shape[-1]):
# Assert that contents are the same
numpy.testing.assert_array_equal(
im_out[..., im_nbr],
self.im_stack[..., im_nbr],
)
def test_download_files(self):
storage_dir = "raw_frames/ML-2005-05-23-10-00-00-0001"
data_storage = s3_storage.S3Storage(storage_dir, self.nbr_workers)
data_storage.upload_frames(self.stack_names, self.im_stack)
data_storage.download_files(
file_names=self.stack_names,
dest_dir=self.temp_path,
)
# Read downloaded file and assert that contents are the same
for i, im_name in enumerate(self.stack_names):
dest_path = os.path.join(self.temp_path, im_name)
im_out = cv2.imread(
dest_path,
cv2.IMREAD_ANYCOLOR | cv2.IMREAD_ANYDEPTH,
)
nose.tools.assert_equal(im_out.dtype, np.uint16)
numpy.testing.assert_array_equal(im_out, self.im_stack[..., i])
def test_upload_frames_color(self):
# Create color image stack
im_stack = np.ones((10, 15, 3, 2), np.uint16) * 3000
im_stack[0:5, 2:4, :, 0] = 42
im_stack[3:7, 12:14, :, 1] = 10000
# Expected color image shape
expected_shape = im_stack[..., 0].shape
# Mock frame upload
storage_dir = "raw_frames/ML-2005-05-23-10-00-00-0001"
data_storage = s3_storage.S3Storage(storage_dir, self.nbr_workers)
data_storage.upload_frames(self.stack_names, im_stack)
# Get images and validate that the contents are unchanged
for im_nbr in range(len(self.stack_names)):
key = "/".join([storage_dir, self.stack_names[im_nbr]])
byte_string = self.conn.Object(self.bucket_name,
key).get()['Body'].read()
# Construct an array from the bytes and decode image
im = im_utils.deserialize_im(byte_string)
# Assert that contents are the same
nose.tools.assert_equal(im.shape, expected_shape)
nose.tools.assert_equal(im.dtype, np.uint16)
numpy.testing.assert_array_equal(im, im_stack[..., im_nbr])
def test_get_stack_from_meta(self):
# Upload image stack
storage_dir = "raw_frames/ML-2005-05-23-10-00-00-0001"
data_storage = s3_storage.S3Storage(storage_dir, self.nbr_workers)
data_storage.upload_frames(self.stack_names, self.im_stack)
global_meta = {
"storage_dir": storage_dir,
"nbr_frames": 2,
"im_height": 10,
"im_width": 15,
"nbr_slices": 1,
"nbr_channels": 2,
"im_colors": 1,
"bit_depth": "uint16",
"nbr_timepoints": 1,
"nbr_positions": 1,
}
frames_meta = meta_utils.make_dataframe(
nbr_frames=global_meta["nbr_frames"], )
nbr_frames = self.im_stack.shape[2]
sha = [None] * nbr_frames
for i in range(nbr_frames):
sha[i] = meta_utils.gen_sha256(self.im_stack[..., i])
frames_meta.loc[0] = [0, 0, 0, "A", "im1.png", 0, sha[0]]
frames_meta.loc[1] = [1, 0, 0, "B", "im2.png", 0, sha[1]]
im_stack, dim_order = data_storage.get_stack_from_meta(
global_meta,
frames_meta,
)
# Stack has X = 10, Y = 15, grayscale, Z = 1, C = 2, T = 1, P = 1
# so expected stack shape and order should be:
expected_shape = (10, 15, 2)
nose.tools.assert_equal(im_stack.shape, expected_shape)
nose.tools.assert_equal(dim_order, "XYC")
def test_existing_storage_path(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
data_storage.upload_file(file_path=self.file_path)
storage_path = os.path.join(self.storage_dir, self.im_name)
self.assertFalse(
data_storage.nonexistent_storage_path(storage_path=storage_path, ))
def test_nonexistent_storage_path(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
storage_path = os.path.join(self.storage_dir, self.im_name)
self.assertTrue(
data_storage.nonexistent_storage_path(storage_path=storage_path, ))
def test_assert_unique_id_exists(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
data_storage.upload_file(file_path=self.file_path)
data_storage.assert_unique_id()
def test_assert_unique_id(self):
data_storage = s3_storage.S3Storage(self.storage_dir, self.nbr_workers)
data_storage.assert_unique_id()