def test_get_bytes_no_file(self) -> None: e = DataFileElement("/not/a/valid/path.txt", readonly=True) # We currently expect, in the case where the filepath doesn't exist, to # get the same bytes as if the file existed and were empty. self.assertEqual(e.get_bytes(), b"") # read-only status should have no effect. e = DataFileElement("/not/a/valid/path.txt", readonly=True) self.assertEqual(e.get_bytes(), b"")
def test_writable_readonly_false(self) -> None: e = DataFileElement('foo') self.assertTrue(e.writable()) e = DataFileElement('foo', False) self.assertTrue(e.writable()) e = DataFileElement('foo', readonly=False) self.assertTrue(e.writable())
def setUpClass(cls): cls.gh_image_fp = os.path.join(TEST_DATA_DIR, "grace_hopper.png") cls.gh_file_element = DataFileElement(cls.gh_image_fp) assert cls.gh_file_element.content_type() == 'image/png' cls.gh_cropped_image_fp = \ os.path.join(TEST_DATA_DIR, 'grace_hopper.100x100+100+100.png') cls.gh_cropped_file_element = DataFileElement(cls.gh_cropped_image_fp) assert cls.gh_cropped_file_element.content_type() == 'image/png' cls.gh_cropped_bbox = AxisAlignedBoundingBox([100, 100], [200, 200])
def setUpClass(cls) -> None: # Initialize test image paths/elements/associated crop boxes. cls.gh_image_fp = os.path.join(TEST_DATA_DIR, "grace_hopper.png") cls.gh_file_element = DataFileElement(cls.gh_image_fp, readonly=True) assert cls.gh_file_element.content_type() == 'image/png' cls.gh_cropped_image_fp = \ os.path.join(TEST_DATA_DIR, 'grace_hopper.100x100+100+100.png') cls.gh_cropped_file_element = \ DataFileElement(cls.gh_cropped_image_fp, readonly=True) assert cls.gh_cropped_file_element.content_type() == 'image/png' cls.gh_cropped_bbox = AxisAlignedBoundingBox([100, 100], [200, 200])
def setup_module(): # Initialize test image paths/elements/associated crop boxes. global GH_IMAGE_FP, GH_FILE_ELEMENT, GH_CROPPED_IMAGE_FP, \ GH_CROPPED_FILE_ELEMENT, GH_CROPPED_BBOX GH_IMAGE_FP = os.path.join(TEST_DATA_DIR, "grace_hopper.png") GH_FILE_ELEMENT = DataFileElement(GH_IMAGE_FP, readonly=True) assert GH_FILE_ELEMENT.content_type() == 'image/png' GH_CROPPED_IMAGE_FP = \ os.path.join(TEST_DATA_DIR, 'grace_hopper.100x100+100+100.png') GH_CROPPED_FILE_ELEMENT = DataFileElement(GH_CROPPED_IMAGE_FP, readonly=True) assert GH_CROPPED_FILE_ELEMENT.content_type() == 'image/png' GH_CROPPED_BBOX = AxisAlignedBoundingBox([100, 100], [200, 200])
def test_load_dataset_tempfile(self) -> None: """ Test DataElement temporary file based context loader. """ # Creating separate element from global so we can mock it up. e = DataFileElement(self.gh_image_fp, readonly=True) e.write_temp = mock.MagicMock(wraps=e.write_temp) # type: ignore e.clean_temp = mock.MagicMock(wraps=e.clean_temp) # type: ignore e.get_bytes = mock.MagicMock(wraps=e.get_bytes) # type: ignore # Using explicit patcher start/stop in order to avoid using ``patch`` # as a decorator because ``osgeo`` might not be defined when # decorating the method. patcher_gdal_open = mock.patch( 'smqtk_image_io.impls.image_reader.gdal_io.gdal.Open', wraps=osgeo.gdal.Open) self.addCleanup(patcher_gdal_open.stop) m_gdal_open = patcher_gdal_open.start() with load_dataset_tempfile(e) as gdal_ds: # noinspection PyUnresolvedReferences e.write_temp.assert_called_once_with() # noinspection PyUnresolvedReferences e.get_bytes.assert_not_called() m_gdal_open.assert_called_once() assert gdal_ds.RasterCount == 3 assert gdal_ds.RasterXSize == 512 assert gdal_ds.RasterYSize == 600 # noinspection PyUnresolvedReferences e.clean_temp.assert_called_once_with() assert len(e._temp_filepath_stack) == 0
def test_repr(self) -> None: e = DataFileElement('foo') self.assertEqual( repr(e), "DataFileElement{filepath: foo, readonly: False, " "explicit_mimetype: None}") e = DataFileElement('bar', readonly=True) self.assertEqual( repr(e), "DataFileElement{filepath: bar, readonly: True, " "explicit_mimetype: None}") e = DataFileElement('baz', readonly=True, explicit_mimetype='some/type') self.assertEqual( repr(e), "DataFileElement{filepath: baz, readonly: True, " "explicit_mimetype: some/type}")
def test_writeTempOverride(self, mock_DataElement_wt: mock.MagicMock) -> None: # no manual directory, should return the base filepath expected_filepath = '/path/to/file.txt' d = DataFileElement(expected_filepath) fp = d.write_temp() self.assertFalse(mock_DataElement_wt.called) self.assertEqual(expected_filepath, fp)
def test_cleanTemp(self) -> None: # a write temp and clean temp should not affect original file source_file = os.path.join(TEST_DATA_DIR, 'test_file.dat') self.assertTrue(os.path.isfile(source_file)) d = DataFileElement(source_file) d.write_temp() self.assertEqual(len(d._temp_filepath_stack), 0) d.clean_temp() self.assertTrue(os.path.isfile(source_file))
def test_configuration(self) -> None: fp = os.path.join(TEST_DATA_DIR, "grace_hopper.png") inst = DataFileElement(filepath=fp, readonly=True, explicit_mimetype='foo/bar') for i in configuration_test_helper(inst): # type: DataFileElement assert i._filepath == fp assert i._readonly is True assert i._explicit_mimetype == 'foo/bar'
def test_set_bytes_writable(self, m_sfw: mock.MagicMock) -> None: # Using a relative filepath test_path = 'foo' test_bytes = b"test string of bytes" e = DataFileElement(test_path) e.set_bytes(test_bytes) # File write function should be called m_sfw.assert_called_once_with(test_path, test_bytes)
def test_writeTempOverride_sameDir( self, mock_DataElement_wt: mock.MagicMock) -> None: expected_filepath = '/path/to/file.txt' target_dir = '/path/to' d = DataFileElement(expected_filepath) fp = d.write_temp(temp_dir=target_dir) self.assertFalse(mock_DataElement_wt.called) self.assertEqual(fp, expected_filepath)
def test_process_load_img(self) -> None: # using image shape, meaning no transformation should occur test_data_layer = 'data' test_transformer = \ caffe.io.Transformer({test_data_layer: (1, 3, 600, 512)}) hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True) a_expected = numpy.asarray(PIL.Image.open(self.hopper_image_fp), numpy.float32) a = _process_load_img_array( (hopper_elem, test_transformer, test_data_layer, False, None)) numpy.testing.assert_allclose(a, a_expected)
def test_load_dataset_vsimem(self): """ Test that VSIMEM loading context """ if LooseVersion(osgeo.__version__).version[0] < 2: pytest.skip("Skipping VSIMEM test because GDAL version < 2") # Creating separate element from global so we can mock it up. e = DataFileElement(GH_IMAGE_FP, readonly=True) e.write_temp = mock.MagicMock(wraps=e.write_temp) e.clean_temp = mock.MagicMock(wraps=e.clean_temp) e.get_bytes = mock.MagicMock(wraps=e.get_bytes) vsimem_path_re = re.compile(r'^/vsimem/\w+$') # Using explicit patcher start/stop in order to avoid using ``patch`` # as a *decorator* because ``osgeo`` might not be defined when # decorating the method. patcher_gdal_open = mock.patch( 'smqtk_image_io.impls.image_reader.gdal_io.gdal.Open', wraps=osgeo.gdal.Open, ) self.addCleanup(patcher_gdal_open.stop) patcher_gdal_unlink = mock.patch( 'smqtk_image_io.impls.image_reader.gdal_io.gdal.Unlink', wraps=osgeo.gdal.Unlink, ) self.addCleanup(patcher_gdal_unlink.stop) m_gdal_open = patcher_gdal_open.start() m_gdal_unlink = patcher_gdal_unlink.start() with load_dataset_vsimem(e) as gdal_ds: # noinspection PyUnresolvedReferences e.write_temp.assert_not_called() # noinspection PyUnresolvedReferences e.get_bytes.assert_called_once_with() m_gdal_open.assert_called_once() ds_path = gdal_ds.GetDescription() assert vsimem_path_re.match(ds_path) assert gdal_ds.RasterCount == 3 assert gdal_ds.RasterXSize == 512 assert gdal_ds.RasterYSize == 600 m_gdal_unlink.assert_called_once_with(ds_path) # noinspection PyUnresolvedReferences e.clean_temp.assert_not_called() assert len(e._temp_filepath_stack) == 0
def test_writeTempOverride_diffDir( self, mock_DataElement_wt: mock.MagicMock) -> None: """ Test that adding ``temp_dir`` parameter triggers call to parent class """ source_filepath = '/path/to/file.png' target_dir = '/some/other/dir' d = DataFileElement(source_filepath) # Should call parent class write_temp since target is not the same dir # that the source file is in. mock_DataElement_wt.return_value = 'expected' v = d.write_temp(temp_dir=target_dir) self.assertEqual(v, 'expected') mock_DataElement_wt.assert_called_with(target_dir)
def test_generate_arrays_dummy_model(self) -> None: # Caffe dummy network interaction test Grace Hopper image) # Construct network with an empty model just to see that our # interaction with the Caffe API is successful. We expect a # zero-valued descriptor vector. g = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem, self.dummy_img_mean_elem, return_layer='fc', use_gpu=False) d_list = list( g._generate_arrays( [DataFileElement(self.hopper_image_fp, readonly=True)])) assert len(d_list) == 1 d = d_list[0] self.assertAlmostEqual(d.sum(), 0., 12)
def test_invalid_datatype(self, _m_cdg_setupNetwork: mock.MagicMock) -> None: # Test that a data element with an incorrect content type for this # implementation raises an exception. # TODO: This probably doesn't need to exist because this is mostly # testing the parent class functionality that should already be # covered by parent class unit tests. # Passing purposefully bag constructor parameters and ignoring # Caffe network setup (above mocking). m_data = mock.MagicMock(spec=DataElement) # noinspection PyTypeChecker g = CaffeDescriptorGenerator(m_data, m_data, None) bad_element = DataFileElement(os.path.join(TEST_DATA_DIR, 'test_file.dat'), readonly=True) # implementation of _generate_arrays checks for self.network value, so # let's set a dummy value to get to the other error condition we want # to actually test. g.network = mock.MagicMock(spec=caffe.Net) with pytest.raises(ValueError): list(g.generate_arrays([bad_element]))
def setup_class(cls) -> None: cls.good_image = DataFileElement( os.path.join(TEST_DATA_DIR, 'Lenna.png')) cls.non_image = DataFileElement( os.path.join(TEST_DATA_DIR, 'test_file.dat'))
def test_init_filepath_abs(self) -> None: fp = '/foo.txt' d = DataFileElement(fp) self.assertEqual(d._filepath, fp)
def test_init_relFilepath_normal(self) -> None: # relative paths should be stored as given within the element fp = 'foo.txt' d = DataFileElement(fp) self.assertEqual(d._filepath, fp)
def test_content_type_explicit_type(self) -> None: ex_type = 'image/png' d = DataFileElement('foo.txt', explicit_mimetype=ex_type) self.assertEqual(d.content_type(), ex_type)
def test_set_bytes_readonly(self) -> None: e = DataFileElement('foo', readonly=True) self.assertRaises(ReadOnlyError, e.set_bytes, b"some bytes")
def test_content_type(self) -> None: d = DataFileElement('foo.txt') self.assertEqual(d.content_type(), 'text/plain')
class TestCaffeDesctriptorGenerator(unittest.TestCase): hopper_image_fp = os.path.join(TEST_DATA_DIR, 'grace_hopper.png') # Dummy Caffe configuration files + weights # - weights is actually an empty file (0 bytes), which caffe treats # as random/zero values (not sure exactly what's happening, but # always results in a zero-vector). dummy_net_topo_elem = DataFileElement(os.path.join( TEST_DATA_DIR, 'caffe.dummpy_network.prototxt'), readonly=True) dummy_caffe_model_elem = DataFileElement(os.path.join( TEST_DATA_DIR, 'caffe.empty_model.caffemodel'), readonly=True) dummy_img_mean_elem = DataFileElement(os.path.join(TEST_DATA_DIR, 'caffe.dummy_mean.npy'), readonly=True) def test_impl_findable(self) -> None: self.assertIn(CaffeDescriptorGenerator, DescriptorGenerator.get_impls()) def test_init_no_prototxt_no_model(self) -> None: """ Test that the class fails to construct and initialize if no network prototext or model are provided. """ with pytest.raises(AttributeError, match="'NoneType' object has no attribute"): # noinspection PyTypeChecker CaffeDescriptorGenerator( network_prototxt=None, # type: ignore network_model=None # type: ignore ) def test_init_no_model(self) -> None: """ Test that the class fails to construct and initialize if only no prototext DataElement is provided. """ with pytest.raises(AttributeError, match="'NoneType' object has no attribute"): # noinspection PyTypeChecker CaffeDescriptorGenerator( network_prototxt=self.dummy_net_topo_elem, network_model=None # type: ignore ) def test_init_no_prototxt(self) -> None: """ Test that the class fails to construct and initialize if only no model DataElement is provided. """ with pytest.raises(AttributeError, match="'NoneType' object has no attribute"): # noinspection PyTypeChecker CaffeDescriptorGenerator( network_prototxt=None, # type: ignore network_model=self.dummy_caffe_model_elem) @mock.patch('smqtk_descriptors.impls.descriptor_generator.caffe1' '.CaffeDescriptorGenerator._setup_network') def test_get_config(self, _m_cdg_setupNetwork: mock.MagicMock) -> None: # Mocking set_network so we don't have to worry about actually # initializing any caffe things for this test. expected_params: Dict[str, Any] = { 'network_prototxt': DataMemoryElement(), 'network_model': DataMemoryElement(), 'image_mean': DataMemoryElement(), 'return_layer': 'layer name', 'batch_size': 777, 'use_gpu': False, 'gpu_device_id': 8, 'network_is_bgr': False, 'data_layer': 'data-other', 'load_truncated_images': True, 'pixel_rescale': (.2, .8), 'input_scale': 1.5, 'threads': 14, } # make sure that we're considering all constructor parameter # options default_params = CaffeDescriptorGenerator.get_default_config() assert set(default_params) == set(expected_params) g = CaffeDescriptorGenerator(**expected_params) # Shift to expecting sub-configs for DataElement params for key in ('network_prototxt', 'network_model', 'image_mean'): expected_params[key] = to_config_dict( cast(DataMemoryElement, expected_params[key])) assert g.get_config() == expected_params @mock.patch('smqtk_descriptors.impls.descriptor_generator.caffe1' '.CaffeDescriptorGenerator._setup_network') def test_config_cycle(self, m_cdg_setup_network: mock.MagicMock) -> None: """ Test being able to get an instances config and use that config to construct an equivalently parameterized instance. This test initializes all possible parameters to non-defaults. """ # Mocking ``_setup_network`` so no caffe functionality is hit during # this test # When every parameter is provided. g1 = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem, image_mean=self.dummy_img_mean_elem, return_layer='foobar', batch_size=9, use_gpu=True, gpu_device_id=99, network_is_bgr=False, data_layer='maybe data', load_truncated_images=True, pixel_rescale=(0.2, 0.3), input_scale=8.9, threads=7) for inst in configuration_test_helper( g1): # type: CaffeDescriptorGenerator assert inst.network_prototxt == self.dummy_net_topo_elem assert inst.network_model == self.dummy_caffe_model_elem assert inst.image_mean == self.dummy_img_mean_elem assert inst.return_layer == 'foobar' assert inst.batch_size == 9 assert inst.use_gpu is True assert inst.gpu_device_id == 99 assert inst.network_is_bgr is False assert inst.data_layer == 'maybe data' assert inst.load_truncated_images is True assert inst.pixel_rescale == (0.2, 0.3) assert inst.input_scale == 8.9 assert inst.threads == 7 @mock.patch('smqtk_descriptors.impls.descriptor_generator.caffe1' '.CaffeDescriptorGenerator._setup_network') def test_config_cycle_imagemean_nonevalued( self, m_cdg_setup_network: mock.MagicMock) -> None: """ Test being able to get an instances config and use that config to construct an equivalently parameterized instance where the second instance is configured with a None-valued 'image_mean' parameter. """ # Mocking ``_setup_network`` so no caffe functionality is hit during # this test # Only required parameters, image_mean is None g1 = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem) g1_config = g1.get_config() # Modify config for g2 to pass None for image_mean for_g2 = dict(g1_config) for_g2['image_mean'] = None g2 = CaffeDescriptorGenerator.from_config(for_g2) expected_config = { 'network_prototxt': to_config_dict(self.dummy_net_topo_elem), 'network_model': to_config_dict(self.dummy_caffe_model_elem), 'image_mean': None, 'return_layer': 'fc7', 'batch_size': 1, 'use_gpu': False, 'gpu_device_id': 0, 'network_is_bgr': True, 'data_layer': 'data', 'load_truncated_images': False, 'pixel_rescale': None, 'input_scale': None, 'threads': None, } assert g1_config == g2.get_config() == expected_config @mock.patch('smqtk_descriptors.impls.descriptor_generator.caffe1' '.CaffeDescriptorGenerator._setup_network') def test_config_cycle_imagemean_nonetyped( self, m_cdg_setup_network: mock.MagicMock) -> None: """ Test being able to get an instances config and use that config to construct an equivalently parameterized instance where the second instance is configured with a None-typed 'image_mean' parameter. """ # Mocking ``_setup_network`` so no caffe functionality is hit during # this test # Only required parameters, image_mean is empty SMQTK configuration # dict g1 = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem) g1_config = g1.get_config() # Modify config for g2 to pass None for image_mean for_g2 = dict(g1_config) for_g2['image_mean'] = {'type': None} g2 = CaffeDescriptorGenerator.from_config(for_g2) expected_config = { 'network_prototxt': to_config_dict(self.dummy_net_topo_elem), 'network_model': to_config_dict(self.dummy_caffe_model_elem), 'image_mean': None, 'return_layer': 'fc7', 'batch_size': 1, 'use_gpu': False, 'gpu_device_id': 0, 'network_is_bgr': True, 'data_layer': 'data', 'load_truncated_images': False, 'pixel_rescale': None, 'input_scale': None, 'threads': None, } assert g1_config == g2.get_config() == expected_config @mock.patch('smqtk_descriptors.impls.descriptor_generator.caffe1' '.CaffeDescriptorGenerator._setup_network') def test_pickle_save_restore(self, m_cdg_setupNetwork: mock.MagicMock) -> None: # Mocking set_network so we don't have to worry about actually # initializing any caffe things for this test. expected_params: Dict[str, Any] = { 'network_prototxt': DataMemoryElement(), 'network_model': DataMemoryElement(), 'image_mean': DataMemoryElement(), 'return_layer': 'layer name', 'batch_size': 777, 'use_gpu': False, 'gpu_device_id': 8, 'network_is_bgr': False, 'data_layer': 'data-other', 'load_truncated_images': True, 'pixel_rescale': (.2, .8), 'input_scale': 1.5, 'threads': 9, } g = CaffeDescriptorGenerator(**expected_params) # Initialization sets up the network on construction. self.assertEqual(m_cdg_setupNetwork.call_count, 1) g_pickled = pickle.dumps(g, -1) g2 = pickle.loads(g_pickled) # Network should be setup for second class class just like in # initial construction. self.assertEqual(m_cdg_setupNetwork.call_count, 2) self.assertIsInstance(g2, CaffeDescriptorGenerator) self.assertEqual(g.get_config(), g2.get_config()) @mock.patch('smqtk_descriptors.impls.descriptor_generator.caffe1' '.CaffeDescriptorGenerator._setup_network') def test_invalid_datatype(self, _m_cdg_setupNetwork: mock.MagicMock) -> None: # Test that a data element with an incorrect content type for this # implementation raises an exception. # TODO: This probably doesn't need to exist because this is mostly # testing the parent class functionality that should already be # covered by parent class unit tests. # Passing purposefully bag constructor parameters and ignoring # Caffe network setup (above mocking). m_data = mock.MagicMock(spec=DataElement) # noinspection PyTypeChecker g = CaffeDescriptorGenerator(m_data, m_data, None) bad_element = DataFileElement(os.path.join(TEST_DATA_DIR, 'test_file.dat'), readonly=True) # implementation of _generate_arrays checks for self.network value, so # let's set a dummy value to get to the other error condition we want # to actually test. g.network = mock.MagicMock(spec=caffe.Net) with pytest.raises(ValueError): list(g.generate_arrays([bad_element])) def test_process_load_img(self) -> None: # using image shape, meaning no transformation should occur test_data_layer = 'data' test_transformer = \ caffe.io.Transformer({test_data_layer: (1, 3, 600, 512)}) hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True) a_expected = numpy.asarray(PIL.Image.open(self.hopper_image_fp), numpy.float32) a = _process_load_img_array( (hopper_elem, test_transformer, test_data_layer, False, None)) numpy.testing.assert_allclose(a, a_expected) def test_generate_arrays_dummy_model(self) -> None: # Caffe dummy network interaction test Grace Hopper image) # Construct network with an empty model just to see that our # interaction with the Caffe API is successful. We expect a # zero-valued descriptor vector. g = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem, self.dummy_img_mean_elem, return_layer='fc', use_gpu=False) d_list = list( g._generate_arrays( [DataFileElement(self.hopper_image_fp, readonly=True)])) assert len(d_list) == 1 d = d_list[0] self.assertAlmostEqual(d.sum(), 0., 12) def test_generate_arrays_no_data(self) -> None: """ Test that generation method correctly returns an empty iterable when no data is passed. """ g = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem, self.dummy_img_mean_elem, return_layer='fc', use_gpu=False) r = list(g._generate_arrays([])) assert r == []
def test_is_empty_file_zero_data(self) -> None: e = DataFileElement(os.path.join(TEST_DATA_DIR, 'test_file.dat')) self.assertTrue(e.is_empty())
def test_get_bytes(self) -> None: # Test with a known real file. test_file_path = os.path.join(TEST_DATA_DIR, 'text_file') e = DataFileElement(test_file_path) self.assertEqual(e.get_bytes(), b"Some text content.\n")
def test_is_empty_file_not_exists(self) -> None: e = DataFileElement('/no/exists') self.assertTrue(e.is_empty())
def test_is_empty_file_has_data(self) -> None: e = DataFileElement(os.path.join(TEST_DATA_DIR, 'grace_hopper.png')) self.assertFalse(e.is_empty())