def consume_file_path(
number: int,
number_1a_path: str,
number_1b_file: str,
number_1c_file_path: str,
number_1d_path_file: str,
number_2a_path: InputPath(str),
number_2b_file: InputPath(str),
number_2c_file_path: InputPath(str),
number_2d_path_file: InputPath(str),
number_3a_path: InputTextFile(str),
number_3b_file: InputTextFile(str),
number_3c_file_path: InputTextFile(str),
number_3d_path_file: InputTextFile(str),
number_4a_path: InputBinaryFile(str),
number_4b_file: InputBinaryFile(str),
number_4c_file_path: InputBinaryFile(str),
number_4d_path_file: InputBinaryFile(str),
output_number_2a_path: OutputPath(str),
output_number_2b_file: OutputPath(str),
output_number_2c_file_path: OutputPath(str),
output_number_2d_path_file: OutputPath(str),
output_number_3a_path: OutputTextFile(str),
output_number_3b_file: OutputTextFile(str),
output_number_3c_file_path: OutputTextFile(str),
output_number_3d_path_file: OutputTextFile(str),
output_number_4a_path: OutputBinaryFile(str),
output_number_4b_file: OutputBinaryFile(str),
output_number_4c_file_path: OutputBinaryFile(str),
output_number_4d_path_file: OutputBinaryFile(str),
):
pass
def train_task(data: InputBinaryFile(str), epochs: int, batch_size: int,
model_path: OutputBinaryFile(str)):
"""Train CNN model on MNIST dataset."""
from tensorflow.python import keras
from tensorflow.python.keras import Sequential, backend as K
from tensorflow.python.keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense
import numpy as np
mnistdata = np.load(data)
train_x = mnistdata['train_x']
train_y = mnistdata['train_y']
test_x = mnistdata['test_x']
test_y = mnistdata['test_y']
num_classes = 10
img_w = 28
img_h = 28
if K.image_data_format() == 'channels_first':
train_x.shape = (-1, 1, img_h, img_w)
test_x.shape = (-1, 1, img_h, img_w)
input_shape = (1, img_h, img_w)
else:
train_x.shape = (-1, img_h, img_w, 1)
test_x.shape = (-1, img_h, img_w, 1)
input_shape = (img_h, img_w, 1)
model = Sequential([
Conv2D(32,
kernel_size=(3, 3),
activation='relu',
input_shape=input_shape),
Conv2D(64, (3, 3), activation='relu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.25),
Flatten(),
Dense(128, activation='relu'),
Dropout(0.5),
Dense(num_classes, activation='softmax'),
])
model.compile(
loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'],
)
model.fit(
train_x,
train_y,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(test_x, test_y),
)
score = model.evaluate(test_x, test_y)
print('Test loss & accuracy: %s' % (score, ))
model.save(model_path)
def write_to_file_path(number_file: OutputBinaryFile(int)):
number_file.write(b'42')
def load_task(
train_images: str,
train_labels: str,
test_images: str,
test_labels: str,
traintest_output: OutputBinaryFile(str),
validation_output: OutputBinaryFile(str),
):
"""Transforms MNIST data from upstream format into numpy array."""
from gzip import GzipFile
from pathlib import Path
from tensorflow.python.keras.utils import get_file
import numpy as np
import struct
from tensorflow.python.keras.utils import to_categorical
def load(path):
"""Ensures that a file is downloaded locally, then unzips and reads it."""
return GzipFile(get_file(Path(path).name, path)).read()
def parse_labels(b: bytes) -> np.array:
"""Parses numeric labels from input data."""
assert struct.unpack('>i', b[:4])[0] == 0x801
return np.frombuffer(b[8:], dtype=np.uint8)
def parse_images(b: bytes) -> np.array:
"""Parses images from input data."""
assert struct.unpack('>i', b[:4])[0] == 0x803
count = struct.unpack('>i', b[4:8])[0]
rows = struct.unpack('>i', b[8:12])[0]
cols = struct.unpack('>i', b[12:16])[0]
data = np.frombuffer(b[16:], dtype=np.uint8)
return data.reshape((count, rows, cols)).astype('float32') / 255
train_x = parse_images(load(train_images))
train_y = to_categorical(parse_labels(load(train_labels)))
test_x = parse_images(load(test_images))
test_y = to_categorical(parse_labels(load(test_labels)))
# For example purposes, we don't need the entire training set, just enough
# to get reasonable accuracy
train_x = train_x[:1000, :, :]
train_y = train_y[:1000]
np.savez_compressed(
traintest_output,
**{
'train_x': train_x,
'train_y': train_y,
'test_x': test_x[100:, :, :],
'test_y': test_y[100:],
},
)
np.savez_compressed(
validation_output,
**{
'val_x': test_x[:100, :, :].reshape(100, 28, 28, 1),
'val_y': test_y[:100]
},
)
