def test_get_closest_locations_returns_closest_single_location_2(self):
generator = DataGenerator()
stops = [MapLocation(3, 3, 3),
MapLocation(4, 4, 4)]
address = MapLocation(5, 5, 5)
closest_stops = generator._get_closest_locations(source=address,
destinations=stops,
n=1)
self.assertEqual(MapLocation(4, 4, 4), closest_stops[0],
"{}, should be 4, 4".format(closest_stops[0]))
def test_get_closest_locations_returns_location_when_equal(self):
generator = DataGenerator()
stops = [MapLocation(1, 1, 1),
MapLocation(3, 3, 3)]
address = MapLocation(2, 2, 2)
closest_stops = generator._get_closest_locations(source=address,
destinations=stops,
n=1)
self.assertEqual(MapLocation(1, 1, 1), closest_stops[0],
"{}, should be 1, 1".format(closest_stops[0]))
def test_get_closest_locations_returns_closest_2_locations(self):
generator = DataGenerator()
stops = [MapLocation(1, 1, 1),
MapLocation(5, 5, 5),
MapLocation(6, 6, 6)]
address = MapLocation(4, 4, 4)
closest_stops = generator._get_closest_locations(source=address,
destinations=stops,
n=2)
self.assertEqual(MapLocation(5, 5, 5), closest_stops[0],
"{}, should be 5, 5".format(closest_stops[0]))
self.assertEqual(MapLocation(6, 6, 6), closest_stops[1],
"{}, should be 6, 6".format(closest_stops[1]))
testType, testNo = 'Test', '01'
repIs = 100
cMatchD, thr_xy = 20, 0.
model_reload, res_recalc = True, False
expTypes, expNo = {'A', 'B'}, {'A': '01', 'B': '01'}
sampleN_test = int(1000)
neuronN = [20]
imageSize, windowSize = [328, 328, 3], [128, 128]
windowStride = [4, 4]
outputSize = {'A': [1], 'B': [50]}
plt.ion()
test_gen = DataGenerator(psf_dir='../../data/exp_psf/psf 512_512.tif',
expType=testType,
sampleN=sampleN_test,
pos_type='rand',
imageSize=imageSize,
show_samples=False)
test_gen.imdb.fixXY = 0.
test_gen.imdb.neuronN = neuronN
test_gen.imdb.xyzis = test_gen.imdb.add_data()
test_gen.dataGeneratorOpts.areaL = [
imageSize[0] - windowSize[0], imageSize[1] - windowSize[1]
]
imgs = np.zeros([repIs, imageSize[0], imageSize[1], imageSize[2]])
for i in range(repIs):
start_num, end_num = i, i + 1
img, _, _ = test_gen.generate_batch(start_num=start_num, end_num=end_num)
imgs[start_num:end_num, :, :, :] = img
x = Conv2D(1024, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
x = GlobalMaxPool2D()(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.5)(x)
predictions = Dense(outputSize[0])(x)
model = Model(inputs=img_input, outputs=predictions)
model.compile(optimizer=SGD(lr=lr, momentum=momentum, decay=decay), loss='hinge', metrics=['accuracy'])
var_sizes = []
for i in range(np.shape(model.layers)[0]):
var_sizes.append(model.layers[i].output_shape)
sampleN_val = max(10, int(sampleN/1000))
train_gen = DataGenerator(sampleN=sampleN, expType=expType, outputSize=outputSize, show_samples=show_samples)
val_gen = DataGenerator(sampleN=sampleN_val, expType=expType, outputSize=outputSize)
##############################
# train model
##############################
hist = model.fit_generator(generator=train_gen.generator(batch_size),
validation_data=val_gen.generator(batch_size),
steps_per_epoch=sampleN/batch_size,
validation_steps=sampleN_val/batch_size,
epochs=nb_epoch, callbacks=[ModelCheckpoint(modelFile)])
##############################
# save model
##############################
model.save(modelFile)
class test_DataGenerator(unittest.TestCase):
def setUp(self):
self.generator = DataGenerator()
# __init__ tests
def test_DataGenerator_exists(self):
self.assertIsInstance(self.generator, DataGenerator)
def test_constructor_sets_stops_array_to_empty_list(self):
self.assertEqual([], self.generator.stops)
def test_constructor_sets_handler_to_simple_handler(self):
self.assertIsInstance(self.generator.handler, DatabaseHandler)
def test_constructor_sets_wrapper_instance_variable(self):
self.assertIsInstance(self.generator.wrapper, MapboxAPIWrapper)
# initialize tests
def test_initialize_initializes_database(self):
self.generator._get_database_handler = Mock()
self.generator.handler.get_all_stops = Mock()
self.generator._get_api_wrapper = Mock()
self.generator.initialize()
self.generator._get_database_handler.assert_called_once_with('db.sqlite3')
def test_initialize_pulls_all_stops(self):
self.generator._get_database_handler = Mock()
self.generator.handler.get_all_stops = Mock()
self.generator._get_api_wrapper = Mock()
self.generator.initialize()
self.generator.handler.get_all_stops.assert_called_once_with()
def test_initialize_initializes_api_wrapper(self):
self.generator._get_database_handler = Mock()
self.generator.handler.get_all_stops = Mock()
self.generator._get_api_wrapper = Mock()
self.generator.initialize()
self.generator._get_api_wrapper.assert_called_once_with('api_key.txt')
# _get_database_handler tests
@patch('DataGeneration.DatabaseHandler.__init__')
def test_get_database_handler_returns_DatabaseHandler(self,
mock_init):
mock_init.return_value = None
self.assertIsInstance(self.generator._get_database_handler(),
DatabaseHandler)
@patch('DataGeneration.DatabaseHandler.__init__')
def test_get_database_handler_inits_DatabaseHandler(self,
mock_init):
mock_init.return_value = None
self.generator._get_database_handler()
mock_init.assert_called_once_with('db.sqlite3')
@patch('DataGeneration.DatabaseHandler.__init__')
def test_get_database_handler_inits_DatabaseHandler_with_db(self,
mock_init):
mock_init.return_value = None
self.generator._get_database_handler(db_file_name='test_db')
mock_init.assert_called_once_with('test_db')
# _get_api_wrapper tests
@patch('DataGeneration.MapboxAPIWrapper.load_api_key_from_file')
def test_get_api_wrapper_returns_MapboxAPIWrapper(self,
mock_get_key):
self.assertIsInstance(self.generator._get_api_wrapper(),
MapboxAPIWrapper)
@patch('DataGeneration.MapboxAPIWrapper.load_api_key_from_file')
def test_get_api_wrapper_pulls_api_key(self,
mock_get_key):
self.generator._get_api_wrapper()
mock_get_key.assert_called_once_with('api_key.txt')
# begin tests
def test_begin_calls_handler_get_next_address(self):
self.generator.handler.get_address_generator = MagicMock()
self.generator.begin(verbose=False)
self.generator.handler.get_address_generator.\
assert_called_once_with(verbose=False)
def test_begin_calls_get_closest_locations(self):
addresses = MapLocation(1, 1, 1)
self.generator.handler.get_address_generator = \
MagicMock(return_value=[addresses])
self.generator.stops = [MapLocation(2, 2, 2), MapLocation(3, 3, 3)]
mock_get_closest_locations = Mock(return_value=[self.generator.stops[0]])
self.generator._get_closest_locations = mock_get_closest_locations
self.generator.wrapper.get_distance_from_api = \
Mock(return_value={"distance": 6, "time": 9})
self.generator.handler.add_route = Mock()
self.generator.begin(stops_per_address=1, verbose=False)
mock_get_closest_locations.assert_called_once_with(addresses,
self.generator.stops,
n=1)
def test_begin_calls_process_stop_for_each_stop(self):
address = MapLocation(1, 1, 1)
self.generator.handler.get_address_generator = \
MagicMock(return_value=[address])
stops = [MapLocation(2, 2, 2), MapLocation(3, 3, 3)]
mock_get_closest_locations = Mock(return_value=stops)
self.generator._get_closest_locations = mock_get_closest_locations
mock_process_stop = Mock()
self.generator.process_stop = mock_process_stop
self.generator.begin(stops_per_address=2, verbose=False)
self.assertEqual(2, mock_process_stop.call_count)
expected_calls = [mock.call(address, stops[0], False),
mock.call(address, stops[1], False)]
self.assertEqual(expected_calls, mock_process_stop.call_args_list)
@patch('DataGeneration.MapboxAPIWrapper.get_distance_from_api')
def test_begin_doesnt_call_add_route_if_MapboxAPIError_occurs(self,
mock_api):
addresses = MapLocation(1, 1, 1)
self.generator.handler.get_address_generator = \
MagicMock(return_value=[addresses])
self.generator.stops = [MapLocation(2, 2, 2)]
self.generator._get_closest_locations = \
Mock(return_value=[self.generator.stops[0]])
# force get_distance_from_api to raise a MapboxAPIError exception
mock_api.side_effect = MapboxAPIError("API Error")
mock_add_route = Mock()
self.generator.handler.add_route = mock_add_route
self.generator.begin(stops_per_address=1, verbose=False)
mock_add_route.assert_not_called()
# process_stop tests
def test_process_stop_calls_get_distance_from_api(self):
mock_get_distance = Mock(return_value={"distance": 6, "time": 9})
self.generator.wrapper.get_distance_from_api = mock_get_distance
self.generator.handler.add_route = Mock()
address = MapLocation(1, 1, 1)
stop = MapLocation(2, 2, 2)
self.generator.process_stop(address=address, stop=stop, verbose=False)
mock_get_distance.assert_called_once_with(address, stop)
def test_process_stop_calls_add_route(self):
self.generator.wrapper.get_distance_from_api = \
Mock(return_value={"distance": 6, "time": 9})
self.generator.handler.add_route = Mock()
self.generator.process_stop(address=MapLocation(3, 3, 3),
stop=MapLocation(4, 4, 4),
verbose=False)
self.generator.handler.add_route.assert_called_once_with(3, 4, 6, 9)
# get_closest_locations tests
def test_get_closest_locations_returns_closest_single_location_1(self):
generator = DataGenerator()
stops = [MapLocation(1, 1, 1),
MapLocation(2, 2, 2)]
address = MapLocation(0, 0, 0)
closest_stops = generator._get_closest_locations(source=address,
destinations=stops,
n=1)
self.assertEqual(MapLocation(1, 1, 1), closest_stops[0],
"{}, should be 1, 1".format(closest_stops[0]))
def test_get_closest_locations_returns_closest_single_location_2(self):
generator = DataGenerator()
stops = [MapLocation(3, 3, 3),
MapLocation(4, 4, 4)]
address = MapLocation(5, 5, 5)
closest_stops = generator._get_closest_locations(source=address,
destinations=stops,
n=1)
self.assertEqual(MapLocation(4, 4, 4), closest_stops[0],
"{}, should be 4, 4".format(closest_stops[0]))
def test_get_closest_locations_returns_closest_2_locations(self):
generator = DataGenerator()
stops = [MapLocation(1, 1, 1),
MapLocation(5, 5, 5),
MapLocation(6, 6, 6)]
address = MapLocation(4, 4, 4)
closest_stops = generator._get_closest_locations(source=address,
destinations=stops,
n=2)
self.assertEqual(MapLocation(5, 5, 5), closest_stops[0],
"{}, should be 5, 5".format(closest_stops[0]))
self.assertEqual(MapLocation(6, 6, 6), closest_stops[1],
"{}, should be 6, 6".format(closest_stops[1]))
def test_get_closest_locations_returns_location_when_equal(self):
generator = DataGenerator()
stops = [MapLocation(1, 1, 1),
MapLocation(3, 3, 3)]
address = MapLocation(2, 2, 2)
closest_stops = generator._get_closest_locations(source=address,
destinations=stops,
n=1)
self.assertEqual(MapLocation(1, 1, 1), closest_stops[0],
"{}, should be 1, 1".format(closest_stops[0]))
def setUp(self):
self.generator = DataGenerator()
# set params
##############################
testType, testNo = 'Test', '01'
repIs = 100
cMatchD, thr_xy = 200, 0.
model_reload, res_recalc = True, False
expTypes, expNo = {'A', 'B'}, {'A': '01', 'B': '01'}
sampleN_test = int(1000)
neuronN = [20]
imageSize, windowSize = [64, 64, 3], [13, 13]
windowStride = [1, 1]
outputSize = [1, 100, 100, 100]
plt.ion()
test_gen = DataGenerator(sampleN=sampleN_test, expType=testType, imageSize=imageSize, outputSize=outputSize, show_samples=False)
imgs = np.zeros([repIs, imageSize[0], imageSize[1], imageSize[2]])
for i in range(repIs):
start_num, end_num = i, i+1
img, _, _, _, _ = test_gen.generate_batch(start_num=start_num, end_num=end_num)
imgs[start_num:end_num, :, :, :] = img
##############################
# set GPU
##############################
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3333)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
for model_type, model_name in model_types.items():
class test_DataGenerator(unittest.TestCase):
def setUp(self):
self.generator = DataGenerator(handler=DatabaseHandler(full=False),
stops=[],
wrapper=MapboxAPIWrapper())
# __init__ tests
def test_DataGenerator_exists(self):
self.assertIsInstance(self.generator, DataGenerator)
def test_constructor_sets_stops_array_to_empty_list(self):
self.assertEqual([], self.generator.stops)
def test_constructor_sets_handler_to_simple_handler(self):
self.assertIsInstance(self.generator.handler, DatabaseHandler)
def test_constructor_sets_wrapper_instance_variable(self):
self.assertIsInstance(self.generator.wrapper, MapboxAPIWrapper)
# initialize tests
def test_initialize_initializes_database(self):
self.generator._get_database_handler = Mock()
self.generator.handler.get_all_stops = Mock()
self.generator._get_api_wrapper = Mock()
self.generator.initialize()
self.generator._get_database_handler.assert_called_once_with(
'db.sqlite3')
def test_initialize_pulls_all_stops(self):
self.generator._get_database_handler = Mock()
self.generator.handler.get_all_stops = Mock()
self.generator._get_api_wrapper = Mock()
self.generator.initialize()
self.generator.handler.get_all_stops.assert_called_once_with()
def test_initialize_initializes_api_wrapper(self):
self.generator._get_database_handler = Mock()
self.generator.handler.get_all_stops = Mock()
self.generator._get_api_wrapper = Mock()
self.generator.initialize()
self.generator._get_api_wrapper.assert_called_once_with('api_key.txt')
# _get_database_handler tests
@patch('DataGeneration.DatabaseHandler.__init__')
def test_get_database_handler_returns_DatabaseHandler(self, mock_init):
mock_init.return_value = None
self.assertIsInstance(self.generator._get_database_handler(),
DatabaseHandler)
@patch('DataGeneration.DatabaseHandler.__init__')
def test_get_database_handler_inits_DatabaseHandler(self, mock_init):
mock_init.return_value = None
self.generator._get_database_handler()
mock_init.assert_called_once_with('db.sqlite3')
@patch('DataGeneration.DatabaseHandler.__init__')
def test_get_database_handler_inits_DatabaseHandler_with_db(
self, mock_init):
mock_init.return_value = None
self.generator._get_database_handler(db_file_name='test_db')
mock_init.assert_called_once_with('test_db')
# _get_api_wrapper tests
@patch('DataGeneration.MapboxAPIWrapper.load_api_key_from_file')
def test_get_api_wrapper_returns_MapboxAPIWrapper(self, mock_get_key):
self.assertIsInstance(self.generator._get_api_wrapper(),
MapboxAPIWrapper)
@patch('DataGeneration.MapboxAPIWrapper.load_api_key_from_file')
def test_get_api_wrapper_pulls_api_key(self, mock_get_key):
self.generator._get_api_wrapper()
mock_get_key.assert_called_once_with('api_key.txt')
# begin tests
def test_begin_calls_handler_get_next_address(self):
self.generator.handler.get_address_generator = MagicMock()
self.generator.begin(verbose=False)
self.generator.handler.get_address_generator.\
assert_called_once_with(verbose=False)
def test_begin_calls_get_closest_locations(self):
addresses = MapLocation(1, 1, 1)
self.generator.handler.get_address_generator = \
MagicMock(return_value=[addresses])
self.generator.stops = [MapLocation(2, 2, 2), MapLocation(3, 3, 3)]
mock_get_closest_locations = Mock(
return_value=[self.generator.stops[0]])
self.generator._get_closest_locations = mock_get_closest_locations
self.generator.wrapper.get_distance_from_api = \
Mock(return_value={"distance": 6, "time": 9})
self.generator.handler.add_route = Mock()
self.generator.begin(stops_per_address=1, verbose=False)
mock_get_closest_locations.assert_called_once_with(
addresses, self.generator.stops, n=1)
def test_begin_calls_process_stop_for_each_stop(self):
address = MapLocation(1, 1, 1)
self.generator.handler.get_address_generator = \
MagicMock(return_value=[address])
stops = [MapLocation(2, 2, 2), MapLocation(3, 3, 3)]
mock_get_closest_locations = Mock(return_value=stops)
self.generator._get_closest_locations = mock_get_closest_locations
mock_process_stop = Mock()
self.generator.process_stop = mock_process_stop
self.generator.begin(stops_per_address=2, verbose=False)
self.assertEqual(2, mock_process_stop.call_count)
expected_calls = [
mock.call(address, stops[0], False, 'walking'),
mock.call(address, stops[1], False, 'walking')
]
self.assertEqual(expected_calls, mock_process_stop.call_args_list)
def test_begin_calls_process_stop_for_each_stop_with_mode(self):
address = MapLocation(1, 1, 1)
self.generator.handler.get_address_generator = \
MagicMock(return_value=[address])
stops = [MapLocation(2, 2, 2), MapLocation(3, 3, 3)]
mock_get_closest_locations = Mock(return_value=stops)
self.generator._get_closest_locations = mock_get_closest_locations
mock_process_stop = Mock()
self.generator.process_stop = mock_process_stop
self.generator.begin(stops_per_address=2,
verbose=False,
mode='driving')
self.assertEqual(2, mock_process_stop.call_count)
expected_calls = [
mock.call(address, stops[0], False, 'driving'),
mock.call(address, stops[1], False, 'driving')
]
self.assertEqual(expected_calls, mock_process_stop.call_args_list)
def test_begin_doesnt_call_add_route_if_MapboxAPIError_occurs(self):
addresses = MapLocation(1, 1, 1)
self.generator.handler.get_address_generator = \
MagicMock(return_value=[addresses])
self.generator.wrapper.get_distance_from_api = Mock()
self.generator.stops = [MapLocation(2, 2, 2)]
self.generator._get_closest_locations = \
Mock(return_value=[self.generator.stops[0]])
# force get_distance_from_api to raise a MapboxAPIError exception
self.generator.wrapper.get_distance_from_api.\
side_effect = MapboxAPIError("API Error")
mock_add_route = Mock()
self.generator.handler.add_route = mock_add_route
self.generator.begin(stops_per_address=1, verbose=False)
mock_add_route.assert_not_called()
# process_stop tests
def test_process_stop_calls_get_distance_from_api(self):
mock_get_distance = Mock(return_value={"distance": 6, "time": 9})
self.generator.wrapper.get_distance_from_api = mock_get_distance
self.generator.handler.add_route = Mock()
address = MapLocation(1, 1, 1)
stop = MapLocation(2, 2, 2)
self.generator.process_stop(address=address, stop=stop, verbose=False)
mock_get_distance.assert_called_once_with(address, stop, 'walking')
def test_process_stop_calls_get_distance_from_api_with_mode(self):
mock_get_distance = Mock(return_value={"distance": 6, "time": 9})
self.generator.wrapper.get_distance_from_api = mock_get_distance
self.generator.handler.add_route = Mock()
address = MapLocation(1, 1, 1)
stop = MapLocation(2, 2, 2)
self.generator.process_stop(address=address,
stop=stop,
verbose=False,
mode='driving')
mock_get_distance.assert_called_once_with(address, stop, 'driving')
def test_process_stop_calls_add_route(self):
self.generator.wrapper.get_distance_from_api = \
Mock(return_value={"distance": 6, "time": 9})
self.generator.handler.add_route = Mock()
self.generator.process_stop(address=MapLocation(3, 3, 3),
stop=MapLocation(4, 4, 4),
verbose=False)
self.generator.handler.add_route.assert_called_once_with(3, 4, 6, 9)
# get_closest_locations tests
def test_get_closest_locations_returns_closest_single_location_1(self):
stops = [MapLocation(1, 1, 1), MapLocation(2, 2, 2)]
address = MapLocation(0, 0, 0)
closest = self.generator._get_closest_locations(source=address,
destinations=stops,
n=1)
self.assertEqual(MapLocation(1, 1, 1), closest[0],
"{}, should be 1, 1".format(closest[0]))
def test_get_closest_locations_returns_closest_single_location_2(self):
stops = [MapLocation(3, 3, 3), MapLocation(4, 4, 4)]
address = MapLocation(5, 5, 5)
closest = self.generator._get_closest_locations(source=address,
destinations=stops,
n=1)
self.assertEqual(MapLocation(4, 4, 4), closest[0],
"{}, should be 4, 4".format(closest[0]))
def test_get_closest_locations_returns_closest_2_locations(self):
stops = [
MapLocation(1, 1, 1),
MapLocation(5, 5, 5),
MapLocation(6, 6, 6)
]
address = MapLocation(4, 4, 4)
closest = self.generator._get_closest_locations(source=address,
destinations=stops,
n=2)
self.assertEqual(MapLocation(5, 5, 5), closest[0],
"{}, should be 5, 5".format(closest[0]))
self.assertEqual(MapLocation(6, 6, 6), closest[1],
"{}, should be 6, 6".format(closest[1]))
def test_get_closest_locations_returns_location_when_equal(self):
stops = [MapLocation(1, 1, 1), MapLocation(3, 3, 3)]
address = MapLocation(2, 2, 2)
closest = self.generator._get_closest_locations(source=address,
destinations=stops,
n=1)
self.assertEqual(MapLocation(1, 1, 1), closest[0],
"{}, should be 1, 1".format(closest[0]))
def setUp(self):
self.generator = DataGenerator(handler=DatabaseHandler(full=False),
stops=[],
wrapper=MapboxAPIWrapper())