def test_model_pipe_batch(self): p = model_util.ModelPipe() p.add(lambda b: [x * x for x in b], batch_size=4) input_list = range(10) output_list = p.apply_ordered(input_list) expected_list = [x * x for x in input_list] self.assertEquals(expected_list, output_list)
def test_model_pipe_exception(self): p = model_util.ModelPipe() p.add(lambda x: 1.0 / x, num_workers=4) input_list = range(-10, 10) output_list = p.apply_ordered(input_list) expected_list = [1.0 / x if x != 0 else None for x in input_list] self.assertEquals(expected_list, output_list)
def test_model_pipe_ordered(self): p = model_util.ModelPipe() p.add(lambda x: x * x, num_workers=32) input_list = range(10) output_list = p.apply_ordered(input_list) expected_list = [x * x for x in input_list] self.assertEquals(expected_list, output_list)
def test_model_pipe_long(self): p = model_util.ModelPipe() p.add(lambda x: x * x, num_workers=32, timeout=0.5) input_dict = {x: x for x in range(10000)} output_dict = p(input_dict) expected_dict = {k: v * v for k, v in input_dict.iteritems()} self.assertEquals(expected_dict, output_dict)
def test_model_pipe(self): p = model_util.ModelPipe() p.add(lambda x: x * x, num_workers=32) input_dict = {x: x for x in range(10)} output_dict = p(input_dict) expected_dict = {k: v * v for k, v in six.iteritems(input_dict)} self.assertEquals(expected_dict, output_dict)
def test_model_pipe_mnist_urls(self): (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train.reshape(60000, 28, 28, 1) x_test = x_test.reshape(10000, 28, 28, 1) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 y_train = to_categorical(y_train, 10) y_test = to_categorical(y_test, 10) model = Sequential() model.add(Flatten(input_shape=(28, 28, 1))) model.add(Dense(128, activation='relu')) model.add(Dense(128, activation='relu')) model.add(Dense(10, activation='softmax')) no_epochs = 3 batch_size = 32 model.compile(loss='categorical_crossentropy', optimizer='adam') model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=no_epochs, batch_size=batch_size) pipe = model_util.ModelPipe() pipe.add(lambda url: urllib.urlopen(url).read(), num_workers=2, timeout=10) pipe.add(lambda img: Image.open(BytesIO(img))) pipe.add(model_util.resize_to_model_input(model)) pipe.add(lambda x: 1 - x) pipe.add(model, num_workers=1, batch_size=32, batcher=np.vstack) pipe.add(lambda x: np.argmax(x, axis=1)) url5 = 'http://blog.otoro.net/assets/20160401/png/mnist_output_10.png' url2 = 'http://joshmontague.com/images/mnist-2.png' urlb = 'http://joshmontague.com/images/mnist-3.png' expected_output = {url5: 5, url2: 2} output = pipe({url5: url5, url2: url2, urlb: urlb}) output = {k: v for k, v in output.iteritems() if v} self.assertEquals(output, expected_output)
def test_model_pipe_keras(self): model = Sequential() model.add(Flatten(input_shape=(1, 28, 28))) model.add(Dense(128, activation='relu')) model.add(Dense(10, activation='softmax')) p = model_util.ModelPipe() input_data = [np.random.random((1, 1, 28, 28)) for _ in range(2)] p.add(model.predict, batch_size=64, batcher=np.vstack) expected_output = [ model.predict(x.reshape((1, 1, 28, 28))) for x in input_data ] output = p.apply_ordered(input_data) self.assertTrue( np.isclose( np.array(output).flatten(), np.array(expected_output).flatten()).all())
write_graph=True, write_images=True) model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=no_epochs, callbacks=[checkpointer, tbcallback], batch_size=batch_size) class_labels = [ 'tshirt/top', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot' ] pipe = model_util.ModelPipe() pipe.add(lambda url: urllib.urlopen(url).read(), num_workers=2) pipe.add(lambda img: Image.open(BytesIO(img))) pipe.add(model_util.resize_to_model_input(model)) pipe.add(lambda x: 1 - x) pipe.add(model, num_workers=1, batch_size=32, batcher=np.vstack) pipe.add(lambda x: np.argmax(x, axis=1)) pipe.add(lambda x: [class_labels[int(x)]]) url_pants = 'https://asda.scene7.com/is/image/Asda/5054622127954_A' url_boot = 'https://images-na.ssl-images-amazon.com/' + \ 'images/I/714sb6gwMpL._UL1500_.jpg' url_broken = 'https://asda.scene7.com/is/image/Asda/5054622127954_B' output = pipe({'pants': url_pants, 'boot': url_boot, 'broken': url_broken}) print output