def setup_class(cls): cls.alexnet_prototxt_elem = DataUrlElement( 'https://data.kitware.com/api/v1/file/57e2f3fd8d777f10f26e532c' '/download') cls.alexnet_caffemodel_elem = DataUrlElement( 'https://data.kitware.com/api/v1/file/57dae22f8d777f10f26a2a86' '/download') cls.image_mean_proto_elem = DataUrlElement( 'https://data.kitware.com/api/v1/file/57dae0a88d777f10f26a2a82' '/download')
def test_configuration(self): default_config = DataUrlElement.get_default_config() ntools.assert_equal(default_config, {'url_address': None}) default_config['url_address'] = EXAMPLE_URL inst1 = DataUrlElement.from_config(default_config) ntools.assert_equal(default_config, inst1.get_config()) ntools.assert_equal(inst1._url, EXAMPLE_URL) inst2 = DataUrlElement.from_config(inst1.get_config()) ntools.assert_equal(inst1, inst2)
def test_from_configuration(self): default_config = DataUrlElement.get_default_config() ntools.assert_equal(default_config, {'url_address': None}) default_config['url_address'] = self.EXAMPLE_URL inst1 = DataUrlElement.from_config(default_config) ntools.assert_equal(default_config, inst1.get_config()) ntools.assert_equal(inst1._url, self.EXAMPLE_URL) ntools.assert_equal(inst1.get_bytes(), open(self.EXAMPLE_PTH).read()) inst2 = DataUrlElement.from_config(inst1.get_config()) ntools.assert_equal(inst1, inst2)
def test_from_configuration(self): default_config = DataUrlElement.get_default_config() self.assertEqual(default_config, {'url_address': None}) default_config['url_address'] = self.EXAMPLE_URL #: :type: DataUrlElement inst1 = DataUrlElement.from_config(default_config) self.assertEqual(default_config, inst1.get_config()) self.assertEqual(inst1._url, self.EXAMPLE_URL) self.assertEqual(inst1.get_bytes(), open(self.EXAMPLE_PTH, 'rb').read()) inst2 = DataUrlElement.from_config(inst1.get_config()) self.assertEqual(inst1, inst2)
def test_set_bytes_universal_readonly(self): # should never be able to write to a url element e = DataUrlElement(self.EXAMPLE_URL) ntools.assert_raises( ReadOnlyError, e.set_bytes, 'foo' )
def test_get_bytes_404_return_code(self, m_requests_get): e = DataUrlElement('some-address') sim_rc = 500 simulated_r = requests.Response() simulated_r.status_code = sim_rc m_requests_get.return_value = simulated_r self.assertRaisesRegexp(requests.HTTPError, '%d' % sim_rc, e.get_bytes)
def resolve_data_element(self, uri): """ Given the URI to some data, resolve it down to a DataElement instance. TODO: More simply use ``smqtk.representation.data_element.from_uri`` function, which should cover these cases as well as others. :raises ValueError: Issue with the given URI regarding either URI source resolution or data resolution. :param uri: URI to data :type uri: str :return: DataElement instance wrapping given URI to data. :rtype: smqtk.representation.DataElement """ self._log.debug("Resolving URI: %s", uri) # Resolve URI into appropriate DataElement instance de: DataElement # declare as to contain the base-class. if uri[:7] == "file://": self._log.debug("Given local disk filepath") filepath = uri[7:] if not os.path.isfile(filepath): raise ValueError("File URI did not point to an existing file " "on disk.") else: de = DataFileElement(filepath) elif uri[:9] == "base64://": self._log.debug("Given base64 string") content_type = flask.request.args.get('content_type', None) self._log.debug("Content type: %s", content_type) if not content_type: raise ValueError("No content-type with given base64 data") else: b64str = uri[9:] de = DataMemoryElement.from_base64(b64str, content_type) else: self._log.debug("Given URL") try: de = DataUrlElement(uri) except requests.HTTPError as ex: raise ValueError("Failed to initialize URL element due to " "HTTPError: %s" % str(ex)) return de
def test_new_from_internet(self): e = DataUrlElement(self.EXAMPLE_URL) self.assertEqual(e.get_bytes(), open(self.EXAMPLE_PTH, 'rb').read())
def test_writable_universal_false(self): # Cannot write to a URL hosted file, should always return false e = DataUrlElement(self.EXAMPLE_URL) self.assertFalse(e.writable())
def test_is_usable(self): # Should always be available, because local/intranet networks are a # thing. self.assertTrue(DataUrlElement.is_usable())
""" Test computing a descriptor on something. """ from smqtk.algorithms.descriptor_generator.caffe_descriptor \ import CaffeDescriptorGenerator from smqtk.representation.data_element.file_element import DataFileElement from smqtk.representation.data_element.url_element import DataUrlElement grace_hopper_img_url = "https://upload.wikimedia.org/wikipedia/commons/5/55/Grace_Hopper.jpg" e = DataUrlElement(grace_hopper_img_url) gen = CaffeDescriptorGenerator( DataFileElement("/home/smqtk/caffe/msra_resnet/ResNet-50-deploy.prototxt"), DataFileElement( "/home/smqtk/caffe/msra_resnet/ResNet-50-model.caffemodel"), DataFileElement("/home/smqtk/caffe/msra_resnet/ResNet_mean.binaryproto"), return_layer="pool5", use_gpu=True, load_truncated_images=True) # Uses default DescriptorMemoryElement factory. d = gen.generate_one_element(e) assert d.vector() is not None print(d.vector())
def test_is_empty_nonzero_bytes(self, _m_requests_get): e = DataUrlElement('some-address') # simulate some content bytes returned e.get_bytes = mock.MagicMock(return_value='some bytes returned') self.assertFalse(e.is_empty())
class TestCaffeDesctriptorGenerator(unittest.TestCase): lenna_image_fp = os.path.join(TEST_DATA_DIR, 'Lenna.png') lenna_alexnet_fc7_descr_fp = \ os.path.join(TEST_DATA_DIR, 'Lenna.alexnet_fc7_output.npy') # Dummy Caffe configuration files + weights # - weights is actually an empty file (0 bytes), which caffe treats # as random/zero values (not sure exactly what's happening, but # always results in a zero-vector). dummy_net_topo_fp = \ os.path.join(TEST_DATA_DIR, 'caffe.dummpy_network.prototxt') dummy_caffe_model_fp = \ os.path.join(TEST_DATA_DIR, 'caffe.empty_model.caffemodel') dummy_img_mean_fp = \ os.path.join(TEST_DATA_DIR, 'caffe.dummy_mean.npy') www_uri_alexnet_prototxt = \ 'https://data.kitware.com/api/v1/file/57e2f3fd8d777f10f26e532c/download' www_uri_alexnet_caffemodel = \ 'https://data.kitware.com/api/v1/file/57dae22f8d777f10f26a2a86/download' www_uri_image_mean_proto = \ 'https://data.kitware.com/api/v1/file/57dae0a88d777f10f26a2a82/download' def test_impl_findable(self): nose.tools.assert_in(CaffeDescriptorGenerator.__name__, get_descriptor_generator_impls()) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_get_config(self, m_cdg_setupNetwork): # Mocking set_network so we don't have to worry about actually # initializing any caffe things for this test. expected_params = { 'network_prototxt_uri': 'some_prototxt_uri', 'network_model_uri': 'some_caffemodel_uri', 'image_mean_uri': 'some_imagemean_uri', 'return_layer': 'layer name', 'batch_size': 777, 'use_gpu': False, 'gpu_device_id': 8, 'network_is_bgr': False, 'data_layer': 'data-other', 'load_truncated_images': True, 'pixel_rescale': (.2, .8), 'input_scale': 1.5, } # make sure that we're considering all constructor parameter options expected_param_keys = \ set(inspect.getargspec(CaffeDescriptorGenerator.__init__) .args[1:]) nose.tools.assert_set_equal(set(expected_params.keys()), expected_param_keys) g = CaffeDescriptorGenerator(**expected_params) nose.tools.assert_equal(g.get_config(), expected_params) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_pickle_save_restore(self, m_cdg_setupNetwork): # Mocking set_network so we don't have to worry about actually # initializing any caffe things for this test. expected_params = { 'network_prototxt_uri': 'some_prototxt_uri', 'network_model_uri': 'some_caffemodel_uri', 'image_mean_uri': 'some_imagemean_uri', 'return_layer': 'layer name', 'batch_size': 777, 'use_gpu': False, 'gpu_device_id': 8, 'network_is_bgr': False, 'data_layer': 'data-other', 'load_truncated_images': True, 'pixel_rescale': (.2, .8), 'input_scale': 1.5, } g = CaffeDescriptorGenerator(**expected_params) # Initialization sets up the network on construction. nose.tools.assert_equal(m_cdg_setupNetwork.call_count, 1) g_pickled = pickle.dumps(g, -1) g2 = pickle.loads(g_pickled) # Network should be setup for second class class just like in # initial construction. nose.tools.assert_equal(m_cdg_setupNetwork.call_count, 2) nose.tools.assert_is_instance(g2, CaffeDescriptorGenerator) nose.tools.assert_equal(g.get_config(), g2.get_config()) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_invalid_datatype(self, m_cdg_setupNetwork): # dummy network setup g = CaffeDescriptorGenerator(None, None, None) bad_element = from_uri(os.path.join(TEST_DATA_DIR, 'test_file.dat')) nose.tools.assert_raises(ValueError, g.compute_descriptor, bad_element) def test_process_load_img(self): # using image shape, meaning no transformation should occur test_data_layer = 'data' test_transformer = \ caffe.io.Transformer({test_data_layer: (1, 3, 512, 512)}) lenna_elem = from_uri(self.lenna_image_fp) a_expected = numpy.asarray(PIL.Image.open(self.lenna_image_fp), numpy.float32) a = _process_load_img_array( (lenna_elem, test_transformer, test_data_layer, None, None)) numpy.testing.assert_allclose(a, a_expected) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_no_internal_compute_descriptor(self, m_cdg_setupNetwork): # This implementation's descriptor computation logic sits in async # method override due to caffe's natural multi-element computation # interface. Thus, ``_compute_descriptor`` should not be # implemented. # dummy network setup because _setup_network is mocked out g = CaffeDescriptorGenerator(0, 0, 0) nose.tools.assert_raises(NotImplementedError, g._compute_descriptor, None) def test_compute_descriptor_dummy_model(self): # Caffe dummy network interaction test Lenna image) # Construct network with an empty model just to see that our # interaction with the Caffe API is successful. We expect a # zero-valued descriptor vector. g = CaffeDescriptorGenerator(self.dummy_net_topo_fp, self.dummy_caffe_model_fp, self.dummy_img_mean_fp, return_layer='fc', use_gpu=False) d = g.compute_descriptor(from_uri(self.lenna_image_fp)) nose.tools.assert_almost_equal(d.vector().sum(), 0., 12) @unittest.skipUnless(DataUrlElement.is_usable(), "URL resolution not functional") def test_compute_descriptor_from_url_lenna_description(self): # Caffe AlexNet interaction test (Lenna image) # This is a long test since it has to download data for remote URIs d = CaffeDescriptorGenerator( self.www_uri_alexnet_prototxt, self.www_uri_alexnet_caffemodel, self.www_uri_image_mean_proto, return_layer='fc7', use_gpu=False, ) lenna_elem = from_uri(self.lenna_image_fp) expected_descr = numpy.load(self.lenna_alexnet_fc7_descr_fp) descr = d.compute_descriptor(lenna_elem).vector() numpy.testing.assert_allclose(descr, expected_descr, atol=1e-5) def test_compute_descriptor_async_no_data(self): # Should get a ValueError when given no descriptors to async method g = CaffeDescriptorGenerator(self.dummy_net_topo_fp, self.dummy_caffe_model_fp, self.dummy_img_mean_fp, return_layer='fc', use_gpu=False) nose.tools.assert_raises(ValueError, g.compute_descriptor_async, [])
def test_is_empty_zero_bytes(self, _m_requests_get): e = DataUrlElement('some-address') # simulate no content bytes returned e.get_bytes = mock.MagicMock(return_value='') self.assertTrue(e.is_empty())
def test_new_add_missing_scheme(self): # Construct without scheme header, should add http:// e = DataUrlElement(self.EXAMPLE_URL[8:]) self.assertEqual(e._url, 'http://' + self.EXAMPLE_URL[8:]) self.assertEqual(e.get_bytes(), open(self.EXAMPLE_PTH, 'rb').read())
def test_new_from_internet(self): e = DataUrlElement(self.EXAMPLE_URL) ntools.assert_equal(e.get_bytes(), open(self.EXAMPLE_PTH).read())
def test_is_empty_zero_bytes(self, m_requests_get): e = DataUrlElement('some-address') # simulate no content bytes returned e.get_bytes = mock.MagicMock(return_value='') ntools.assert_true(e.is_empty())
def test_configuration(self, m_req_get): # Mocking requests usage to no actually head into the network. inst = DataUrlElement(self.EXAMPLE_URL) for i in configuration_test_helper(inst): # type: DataUrlElement assert i._url == self.EXAMPLE_URL
class TestCaffeDesctriptorGenerator(unittest.TestCase): hopper_image_fp = os.path.join(TEST_DATA_DIR, 'grace_hopper.png') hopper_alexnet_fc7_descr_fp = os.path.join( TEST_DATA_DIR, 'Hopper.alexnet_fc7_output.npy') # Dummy Caffe configuration files + weights # - weights is actually an empty file (0 bytes), which caffe treats # as random/zero values (not sure exactly what's happening, but # always results in a zero-vector). dummy_net_topo_elem = DataFileElement(os.path.join( TEST_DATA_DIR, 'caffe.dummpy_network.prototxt'), readonly=True) dummy_caffe_model_elem = DataFileElement(os.path.join( TEST_DATA_DIR, 'caffe.empty_model.caffemodel'), readonly=True) dummy_img_mean_elem = DataFileElement(os.path.join(TEST_DATA_DIR, 'caffe.dummy_mean.npy'), readonly=True) @classmethod def setup_class(cls): cls.alexnet_prototxt_elem = DataUrlElement( 'https://data.kitware.com/api/v1/file/57e2f3fd8d777f10f26e532c' '/download') cls.alexnet_caffemodel_elem = DataUrlElement( 'https://data.kitware.com/api/v1/file/57dae22f8d777f10f26a2a86' '/download') cls.image_mean_proto_elem = DataUrlElement( 'https://data.kitware.com/api/v1/file/57dae0a88d777f10f26a2a82' '/download') def test_impl_findable(self): self.assertIn(CaffeDescriptorGenerator, DescriptorGenerator.get_impls()) def test_init_no_prototxt_no_model(self): """ Test that the class fails to construct and initialize if no network prototext or model are provided. """ with pytest.raises(AttributeError, match="'NoneType' object has no attribute"): # noinspection PyTypeChecker CaffeDescriptorGenerator(network_prototxt=None, network_model=None) def test_init_no_model(self): """ Test that the class fails to construct and initialize if only no prototext DataElement is provided. """ with pytest.raises(AttributeError, match="'NoneType' object has no attribute"): # noinspection PyTypeChecker CaffeDescriptorGenerator(network_prototxt=self.dummy_net_topo_elem, network_model=None) def test_init_no_prototxt(self): """ Test that the class fails to construct and initialize if only no model DataElement is provided. """ with pytest.raises(AttributeError, match="'NoneType' object has no attribute"): # noinspection PyTypeChecker CaffeDescriptorGenerator(network_prototxt=None, network_model=self.dummy_caffe_model_elem) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_get_config(self, _m_cdg_setupNetwork): # Mocking set_network so we don't have to worry about actually # initializing any caffe things for this test. expected_params = { 'network_prototxt': DataMemoryElement(), 'network_model': DataMemoryElement(), 'image_mean': DataMemoryElement(), 'return_layer': 'layer name', 'batch_size': 777, 'use_gpu': False, 'gpu_device_id': 8, 'network_is_bgr': False, 'data_layer': 'data-other', 'load_truncated_images': True, 'pixel_rescale': (.2, .8), 'input_scale': 1.5, 'threads': 14, } # make sure that we're considering all constructor parameter # options default_params = CaffeDescriptorGenerator.get_default_config() assert set(default_params) == set(expected_params) g = CaffeDescriptorGenerator(**expected_params) # Shift to expecting sub-configs for DataElement params for key in ('network_prototxt', 'network_model', 'image_mean'): expected_params[key] = to_config_dict(expected_params[key]) assert g.get_config() == expected_params @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_config_cycle(self, m_cdg_setup_network): """ Test being able to get an instances config and use that config to construct an equivalently parameterized instance. This test initializes all possible parameters to non-defaults. """ # Mocking ``_setup_network`` so no caffe functionality is hit during # this test # When every parameter is provided. g1 = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem, image_mean=self.dummy_img_mean_elem, return_layer='foobar', batch_size=9, use_gpu=True, gpu_device_id=99, network_is_bgr=False, data_layer='maybe data', load_truncated_images=True, pixel_rescale=(0.2, 0.3), input_scale=8.9, threads=7) for inst in configuration_test_helper( g1): # type: CaffeDescriptorGenerator assert inst.network_prototxt == self.dummy_net_topo_elem assert inst.network_model == self.dummy_caffe_model_elem assert inst.image_mean == self.dummy_img_mean_elem assert inst.return_layer == 'foobar' assert inst.batch_size == 9 assert inst.use_gpu == True assert inst.gpu_device_id == 99 assert inst.network_is_bgr == False assert inst.data_layer == 'maybe data' assert inst.load_truncated_images == True assert inst.pixel_rescale == (0.2, 0.3) assert inst.input_scale == 8.9 assert inst.threads == 7 @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_config_cycle_imagemean_nonevalued(self, m_cdg_setup_network): """ Test being able to get an instances config and use that config to construct an equivalently parameterized instance where the second instance is configured with a None-valued 'image_mean' parameter. """ # Mocking ``_setup_network`` so no caffe functionality is hit during # this test # Only required parameters, image_mean is None g1 = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem) g1_config = g1.get_config() # Modify config for g2 to pass None for image_mean for_g2 = dict(g1_config) for_g2['image_mean'] = None g2 = CaffeDescriptorGenerator.from_config(for_g2) expected_config = { 'network_prototxt': to_config_dict(self.dummy_net_topo_elem), 'network_model': to_config_dict(self.dummy_caffe_model_elem), 'image_mean': None, 'return_layer': 'fc7', 'batch_size': 1, 'use_gpu': False, 'gpu_device_id': 0, 'network_is_bgr': True, 'data_layer': 'data', 'load_truncated_images': False, 'pixel_rescale': None, 'input_scale': None, 'threads': None, } assert g1_config == g2.get_config() == expected_config @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_config_cycle_imagemean_nonetyped(self, m_cdg_setup_network): """ Test being able to get an instances config and use that config to construct an equivalently parameterized instance where the second instance is configured with a None-typed 'image_mean' parameter. """ # Mocking ``_setup_network`` so no caffe functionality is hit during # this test # Only required parameters, image_mean is empty SMQTK configuration # dict g1 = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem) g1_config = g1.get_config() # Modify config for g2 to pass None for image_mean for_g2 = dict(g1_config) for_g2['image_mean'] = {'type': None} g2 = CaffeDescriptorGenerator.from_config(for_g2) expected_config = { 'network_prototxt': to_config_dict(self.dummy_net_topo_elem), 'network_model': to_config_dict(self.dummy_caffe_model_elem), 'image_mean': None, 'return_layer': 'fc7', 'batch_size': 1, 'use_gpu': False, 'gpu_device_id': 0, 'network_is_bgr': True, 'data_layer': 'data', 'load_truncated_images': False, 'pixel_rescale': None, 'input_scale': None, 'threads': None, } assert g1_config == g2.get_config() == expected_config @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_pickle_save_restore(self, m_cdg_setupNetwork): # Mocking set_network so we don't have to worry about actually # initializing any caffe things for this test. expected_params = { 'network_prototxt': DataMemoryElement(), 'network_model': DataMemoryElement(), 'image_mean': DataMemoryElement(), 'return_layer': 'layer name', 'batch_size': 777, 'use_gpu': False, 'gpu_device_id': 8, 'network_is_bgr': False, 'data_layer': 'data-other', 'load_truncated_images': True, 'pixel_rescale': (.2, .8), 'input_scale': 1.5, 'threads': 9, } g = CaffeDescriptorGenerator(**expected_params) # Initialization sets up the network on construction. self.assertEqual(m_cdg_setupNetwork.call_count, 1) g_pickled = pickle.dumps(g, -1) g2 = pickle.loads(g_pickled) # Network should be setup for second class class just like in # initial construction. self.assertEqual(m_cdg_setupNetwork.call_count, 2) self.assertIsInstance(g2, CaffeDescriptorGenerator) self.assertEqual(g.get_config(), g2.get_config()) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_invalid_datatype(self, _m_cdg_setupNetwork): # Test that a data element with an incorrect content type for this # implementation raises an exception. # TODO: This probably doesn't need to exist because this is mostly # testing the parent class functionality that should already be # covered by parent class unit tests. # Passing purposefully bag constructor parameters and ignoring # Caffe network setup (above mocking). # noinspection PyTypeChecker g = CaffeDescriptorGenerator(None, None, None) bad_element = DataFileElement(os.path.join(TEST_DATA_DIR, 'test_file.dat'), readonly=True) with pytest.raises(ValueError): list(g.generate_arrays([bad_element])) def test_process_load_img(self): # using image shape, meaning no transformation should occur test_data_layer = 'data' test_transformer = \ caffe.io.Transformer({test_data_layer: (1, 3, 600, 512)}) hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True) a_expected = numpy.asarray(PIL.Image.open(self.hopper_image_fp), numpy.float32) a = _process_load_img_array( (hopper_elem, test_transformer, test_data_layer, None, None)) numpy.testing.assert_allclose(a, a_expected) def test_generate_arrays_dummy_model(self): # Caffe dummy network interaction test Grace Hopper image) # Construct network with an empty model just to see that our # interaction with the Caffe API is successful. We expect a # zero-valued descriptor vector. g = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem, self.dummy_img_mean_elem, return_layer='fc', use_gpu=False) d_list = list( g._generate_arrays( [DataFileElement(self.hopper_image_fp, readonly=True)])) assert len(d_list) == 1 d = d_list[0] self.assertAlmostEqual(d.sum(), 0., 12) def test_generate_arrays_no_data(self): """ Test that generation method correctly returns an empty iterable when no data is passed. """ g = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem, self.dummy_img_mean_elem, return_layer='fc', use_gpu=False) r = list(g._generate_arrays([])) assert r == [] @unittest.skipUnless(DataUrlElement.is_usable(), "URL resolution not functional") def test_generate_arrays_from_url_hopper_description(self): # Caffe AlexNet interaction test (Grace Hopper image) # This is a long test since it has to download data for remote URIs d = CaffeDescriptorGenerator( self.alexnet_prototxt_elem, self.alexnet_caffemodel_elem, self.image_mean_proto_elem, return_layer='fc7', use_gpu=False, ) hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True) expected_descr = numpy.load(self.hopper_alexnet_fc7_descr_fp) descr_list = list(d._generate_arrays([hopper_elem])) assert len(descr_list) == 1 descr = descr_list[0] numpy.testing.assert_allclose(descr, expected_descr, atol=1e-4)
def test_content_type(self): e = DataUrlElement(self.EXAMPLE_URL) self.assertEqual(e.content_type(), 'image/png')
def test_get_bytes_from_url(self): e = DataUrlElement(self.EXAMPLE_URL) ntools.assert_equal(e.get_bytes(), open(self.EXAMPLE_PTH, 'rb').read()) ntools.assert_equal(e.content_type(), 'image/png')
def test_from_uri(self): e = DataUrlElement.from_uri(self.EXAMPLE_URL) self.assertEqual(e.get_bytes(), open(self.EXAMPLE_PTH, 'rb').read()) self.assertEqual(e.content_type(), 'image/png')
class TestCaffeDesctriptorGenerator(unittest.TestCase): hopper_image_fp = get_sample_data('grace_hopper.png', asfileobj=False) hopper_alexnet_fc7_descr_fp = os.path.join( TEST_DATA_DIR, 'Hopper.alexnet_fc7_output.npy') # Dummy Caffe configuration files + weights # - weights is actually an empty file (0 bytes), which caffe treats # as random/zero values (not sure exactly what's happening, but # always results in a zero-vector). dummy_net_topo_elem = DataFileElement(os.path.join( TEST_DATA_DIR, 'caffe.dummpy_network.prototxt'), readonly=True) dummy_caffe_model_elem = DataFileElement(os.path.join( TEST_DATA_DIR, 'caffe.empty_model.caffemodel'), readonly=True) dummy_img_mean_elem = DataFileElement(os.path.join( TEST_DATA_DIR, 'caffe.dummy_mean.npy'), readonly=True) @classmethod def setup_class(cls): cls.alexnet_prototxt_elem = DataUrlElement( 'https://data.kitware.com/api/v1/file/57e2f3fd8d777f10f26e532c' '/download') cls.alexnet_caffemodel_elem = DataUrlElement( 'https://data.kitware.com/api/v1/file/57dae22f8d777f10f26a2a86' '/download') cls.image_mean_proto_elem = DataUrlElement( 'https://data.kitware.com/api/v1/file/57dae0a88d777f10f26a2a82' '/download') def test_impl_findable(self): self.assertIn(CaffeDescriptorGenerator, DescriptorGenerator.get_impls()) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_get_config(self, _m_cdg_setupNetwork): # Mocking set_network so we don't have to worry about actually # initializing any caffe things for this test. expected_params = { 'network_prototxt': DataMemoryElement(), 'network_model': DataMemoryElement(), 'image_mean': DataMemoryElement(), 'return_layer': 'layer name', 'batch_size': 777, 'use_gpu': False, 'gpu_device_id': 8, 'network_is_bgr': False, 'data_layer': 'data-other', 'load_truncated_images': True, 'pixel_rescale': (.2, .8), 'input_scale': 1.5, } # make sure that we're considering all constructor parameter options expected_param_keys = \ set(inspect.getargspec(CaffeDescriptorGenerator.__init__) .args[1:]) self.assertSetEqual(set(expected_params.keys()), expected_param_keys) g = CaffeDescriptorGenerator(**expected_params) for key in ('network_prototxt', 'network_model', 'image_mean'): expected_params[key] = to_config_dict(expected_params[key]) self.assertEqual(g.get_config(), expected_params) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_pickle_save_restore(self, m_cdg_setupNetwork): # Mocking set_network so we don't have to worry about actually # initializing any caffe things for this test. expected_params = { 'network_prototxt': DataMemoryElement(), 'network_model': DataMemoryElement(), 'image_mean': DataMemoryElement(), 'return_layer': 'layer name', 'batch_size': 777, 'use_gpu': False, 'gpu_device_id': 8, 'network_is_bgr': False, 'data_layer': 'data-other', 'load_truncated_images': True, 'pixel_rescale': (.2, .8), 'input_scale': 1.5, } g = CaffeDescriptorGenerator(**expected_params) # Initialization sets up the network on construction. self.assertEqual(m_cdg_setupNetwork.call_count, 1) g_pickled = pickle.dumps(g, -1) g2 = pickle.loads(g_pickled) # Network should be setup for second class class just like in # initial construction. self.assertEqual(m_cdg_setupNetwork.call_count, 2) self.assertIsInstance(g2, CaffeDescriptorGenerator) self.assertEqual(g.get_config(), g2.get_config()) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_invalid_datatype(self, _m_cdg_setupNetwork): # Test that a data element with an incorrect content type raises an # exception. # Passing purposefully bag constructor parameters and ignoring # Caffe network setup (above mocking). # noinspection PyTypeChecker g = CaffeDescriptorGenerator(None, None, None) bad_element = DataFileElement(os.path.join(TEST_DATA_DIR, 'test_file.dat'), readonly=True) self.assertRaises(ValueError, g.compute_descriptor, bad_element) def test_process_load_img(self): # using image shape, meaning no transformation should occur test_data_layer = 'data' test_transformer = \ caffe.io.Transformer({test_data_layer: (1, 3, 600, 512)}) hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True) a_expected = numpy.asarray(PIL.Image.open(self.hopper_image_fp), numpy.float32) a = _process_load_img_array( (hopper_elem, test_transformer, test_data_layer, None, None)) numpy.testing.assert_allclose(a, a_expected) @mock.patch('smqtk.algorithms.descriptor_generator.caffe_descriptor' '.CaffeDescriptorGenerator._setup_network') def test_no_internal_compute_descriptor(self, _m_cdg_setupNetwork): # This implementation's descriptor computation logic sits in async # method override due to caffe's natural multi-element computation # interface. Thus, ``_compute_descriptor`` should not be # implemented. # Passing purposefully bag constructor parameters and ignoring # Caffe network setup (above mocking). # noinspection PyTypeChecker g = CaffeDescriptorGenerator(0, 0, 0) self.assertRaises(NotImplementedError, g._compute_descriptor, None) def test_compute_descriptor_dummy_model(self): # Caffe dummy network interaction test Lenna image) # Construct network with an empty model just to see that our # interaction with the Caffe API is successful. We expect a # zero-valued descriptor vector. g = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem, self.dummy_img_mean_elem, return_layer='fc', use_gpu=False) d = g.compute_descriptor( DataFileElement(self.hopper_image_fp, readonly=True)) self.assertAlmostEqual(d.vector().sum(), 0., 12) @unittest.skipUnless(DataUrlElement.is_usable(), "URL resolution not functional") def test_compute_descriptor_from_url_hopper_description(self): # Caffe AlexNet interaction test (Grace Hopper image) # This is a long test since it has to download data for remote URIs d = CaffeDescriptorGenerator( self.alexnet_prototxt_elem, self.alexnet_caffemodel_elem, self.image_mean_proto_elem, return_layer='fc7', use_gpu=False, ) hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True) expected_descr = numpy.load(self.hopper_alexnet_fc7_descr_fp) descr = d.compute_descriptor(hopper_elem).vector() numpy.testing.assert_allclose(descr, expected_descr, atol=1e-4) def test_compute_descriptor_async_no_data(self): # Should get a ValueError when given no descriptors to async method g = CaffeDescriptorGenerator(self.dummy_net_topo_elem, self.dummy_caffe_model_elem, self.dummy_img_mean_elem, return_layer='fc', use_gpu=False) self.assertRaises(ValueError, g.compute_descriptor_async, [])