def test_model_pipe_batch(self):
p = model_util.ModelPipe()
p.add(lambda b: [x * x for x in b], batch_size=4)
input_list = range(10)
output_list = p.apply_ordered(input_list)
expected_list = [x * x for x in input_list]
self.assertEquals(expected_list, output_list)
def test_model_pipe_exception(self):
p = model_util.ModelPipe()
p.add(lambda x: 1.0 / x, num_workers=4)
input_list = range(-10, 10)
output_list = p.apply_ordered(input_list)
expected_list = [1.0 / x if x != 0 else None for x in input_list]
self.assertEquals(expected_list, output_list)
def test_model_pipe_ordered(self):
p = model_util.ModelPipe()
p.add(lambda x: x * x, num_workers=32)
input_list = range(10)
output_list = p.apply_ordered(input_list)
expected_list = [x * x for x in input_list]
self.assertEquals(expected_list, output_list)
def test_model_pipe_long(self):
p = model_util.ModelPipe()
p.add(lambda x: x * x, num_workers=32, timeout=0.5)
input_dict = {x: x for x in range(10000)}
output_dict = p(input_dict)
expected_dict = {k: v * v for k, v in input_dict.iteritems()}
self.assertEquals(expected_dict, output_dict)
def test_model_pipe(self):
p = model_util.ModelPipe()
p.add(lambda x: x * x, num_workers=32)
input_dict = {x: x for x in range(10)}
output_dict = p(input_dict)
expected_dict = {k: v * v for k, v in six.iteritems(input_dict)}
self.assertEquals(expected_dict, output_dict)
def test_model_pipe_mnist_urls(self):
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 28, 28, 1)
x_test = x_test.reshape(10000, 28, 28, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
y_train = to_categorical(y_train, 10)
y_test = to_categorical(y_test, 10)
model = Sequential()
model.add(Flatten(input_shape=(28, 28, 1)))
model.add(Dense(128, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dense(10, activation='softmax'))
no_epochs = 3
batch_size = 32
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.fit(x_train,
y_train,
validation_data=(x_test, y_test),
epochs=no_epochs,
batch_size=batch_size)
pipe = model_util.ModelPipe()
pipe.add(lambda url: urllib.urlopen(url).read(),
num_workers=2,
timeout=10)
pipe.add(lambda img: Image.open(BytesIO(img)))
pipe.add(model_util.resize_to_model_input(model))
pipe.add(lambda x: 1 - x)
pipe.add(model, num_workers=1, batch_size=32, batcher=np.vstack)
pipe.add(lambda x: np.argmax(x, axis=1))
url5 = 'http://blog.otoro.net/assets/20160401/png/mnist_output_10.png'
url2 = 'http://joshmontague.com/images/mnist-2.png'
urlb = 'http://joshmontague.com/images/mnist-3.png'
expected_output = {url5: 5, url2: 2}
output = pipe({url5: url5, url2: url2, urlb: urlb})
output = {k: v for k, v in output.iteritems() if v}
self.assertEquals(output, expected_output)
def test_model_pipe_keras(self):
model = Sequential()
model.add(Flatten(input_shape=(1, 28, 28)))
model.add(Dense(128, activation='relu'))
model.add(Dense(10, activation='softmax'))
p = model_util.ModelPipe()
input_data = [np.random.random((1, 1, 28, 28)) for _ in range(2)]
p.add(model.predict, batch_size=64, batcher=np.vstack)
expected_output = [
model.predict(x.reshape((1, 1, 28, 28))) for x in input_data
]
output = p.apply_ordered(input_data)
self.assertTrue(
np.isclose(
np.array(output).flatten(),
np.array(expected_output).flatten()).all())
write_graph=True,
write_images=True)
model.fit(x_train,
y_train,
validation_data=(x_test, y_test),
epochs=no_epochs,
callbacks=[checkpointer, tbcallback],
batch_size=batch_size)
class_labels = [
'tshirt/top', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt',
'sneaker', 'bag', 'ankle boot'
]
pipe = model_util.ModelPipe()
pipe.add(lambda url: urllib.urlopen(url).read(), num_workers=2)
pipe.add(lambda img: Image.open(BytesIO(img)))
pipe.add(model_util.resize_to_model_input(model))
pipe.add(lambda x: 1 - x)
pipe.add(model, num_workers=1, batch_size=32, batcher=np.vstack)
pipe.add(lambda x: np.argmax(x, axis=1))
pipe.add(lambda x: [class_labels[int(x)]])
url_pants = 'https://asda.scene7.com/is/image/Asda/5054622127954_A'
url_boot = 'https://images-na.ssl-images-amazon.com/' + \
'images/I/714sb6gwMpL._UL1500_.jpg'
url_broken = 'https://asda.scene7.com/is/image/Asda/5054622127954_B'
output = pipe({'pants': url_pants, 'boot': url_boot, 'broken': url_broken})
print output
