def test_download_pts(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
positions=0,
times=0,
slices=1,
)
meta_path = os.path.join(
dest_dir,
self.dataset_serial,
'global_metadata.json',
)
frames_meta = pd.read_csv(meta_path)
for i, row in frames_meta.iterrows():
self.assertEqual(row.pos_idx, 0)
self.assertEqual(row.time_idx, 0)
self.assertEqual(row.slice_idx, 1)
def test_download_channel(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
channels=1,
)
download_dir = os.path.join(dest_dir, self.dataset_serial)
# Check frames_meta content
frames_meta = pd.read_csv(os.path.join(download_dir,
'frames_meta.csv'))
for i, row in frames_meta.iterrows():
self.assertEqual(row.channel_idx, 1)
im_name = 'im_c001_z00{}_t000_p000.png'.format(i)
self.assertEqual(row.file_name, im_name)
# Check downloaded images
im_order = [1, 3, 5]
for z in range(3):
im_name = 'im_c001_z00{}_t000_p000.png'.format(z)
im_path = os.path.join(download_dir, im_name)
im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
numpy.testing.assert_array_equal(im, self.im[im_order[i], ...])
def test_download_frames(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
)
# Images are separated by slice first then channel
im_order = [0, 2, 4, 1, 3, 5]
it = itertools.product(range(self.nbr_channels),
range(self.nbr_slices))
for i, (c, z) in enumerate(it):
im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z)
im_path = os.path.join(
dest_dir,
self.dataset_serial,
im_name,
)
im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
numpy.testing.assert_array_equal(im, self.im[im_order[i], ...])
# Read and validate frames meta
meta_path = os.path.join(
dest_dir,
self.dataset_serial,
'frames_meta.csv',
)
frames_meta = pd.read_csv(meta_path)
for i, row in frames_meta.iterrows():
c = i // self.nbr_slices
z = i % self.nbr_slices
self.assertEqual(row.channel_idx, c)
self.assertEqual(row.slice_idx, z)
self.assertEqual(row.time_idx, 0)
self.assertEqual(row.pos_idx, 0)
im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z)
self.assertEqual(row.file_name, im_name)
sha256 = meta_utils.gen_sha256(self.im[im_order[i], ...])
self.assertEqual(row.sha256, sha256)
# Read and validate global meta
meta_path = os.path.join(
dest_dir,
self.dataset_serial,
'global_metadata.json',
)
meta_json = json_ops.read_json_file(meta_path)
self.assertEqual(meta_json['storage_dir'], self.frames_storage_dir)
self.assertEqual(meta_json['nbr_frames'], 6)
self.assertEqual(meta_json['im_width'], 15)
self.assertEqual(meta_json['im_height'], 10)
self.assertEqual(meta_json['nbr_slices'], self.nbr_slices)
self.assertEqual(meta_json['nbr_channels'], self.nbr_channels)
self.assertEqual(meta_json['im_colors'], 1)
self.assertEqual(meta_json['nbr_timepoints'], 1)
self.assertEqual(meta_json['nbr_positions'], 1)
self.assertEqual(meta_json['bit_depth'], 'uint16')
def test_download_channel_name(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
channels='channel1',
)
def test_negative_workers(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
data_downloader.download_data(
dataset_serial=self.dataset_serial_file,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
metadata=False,
download=False,
nbr_workers=-2,
)
def test_folder_exists(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
self.tempdir.makedir(
os.path.join('dest_dir', self.dataset_serial_file), )
dest_dir = os.path.join(self.temp_path, 'dest_dir')
data_downloader.download_data(
dataset_serial=self.dataset_serial_file,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
nbr_workers=2,
metadata=False,
)
def test_invalid_dataset(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
self.tempdir.makedir(
os.path.join('dest_dir', self.dataset_serial_file), )
dest_dir = os.path.join(self.temp_path, 'dest_dir')
data_downloader.download_data(
dataset_serial='Not-a-serial',
login=self.credentials_path,
dest=dest_dir,
storage='s3',
metadata=False,
nbr_workers=2,
)
def test_download_file(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial_file,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
metadata=False,
nbr_workers=2,
)
# See if file has been downloaded
file_path = os.path.join(
dest_dir,
self.dataset_serial_file,
'*',
)
found_file = os.path.basename(glob.glob(file_path)[0])
self.assertEqual("A1_2_PROTEIN_test.tif", found_file)
