def consume_file_path( number: int, number_1a_path: str, number_1b_file: str, number_1c_file_path: str, number_1d_path_file: str, number_2a_path: InputPath(str), number_2b_file: InputPath(str), number_2c_file_path: InputPath(str), number_2d_path_file: InputPath(str), number_3a_path: InputTextFile(str), number_3b_file: InputTextFile(str), number_3c_file_path: InputTextFile(str), number_3d_path_file: InputTextFile(str), number_4a_path: InputBinaryFile(str), number_4b_file: InputBinaryFile(str), number_4c_file_path: InputBinaryFile(str), number_4d_path_file: InputBinaryFile(str), output_number_2a_path: OutputPath(str), output_number_2b_file: OutputPath(str), output_number_2c_file_path: OutputPath(str), output_number_2d_path_file: OutputPath(str), output_number_3a_path: OutputTextFile(str), output_number_3b_file: OutputTextFile(str), output_number_3c_file_path: OutputTextFile(str), output_number_3d_path_file: OutputTextFile(str), output_number_4a_path: OutputBinaryFile(str), output_number_4b_file: OutputBinaryFile(str), output_number_4c_file_path: OutputBinaryFile(str), output_number_4d_path_file: OutputBinaryFile(str), ): pass
def train_task(data: InputBinaryFile(str), epochs: int, batch_size: int, model_path: OutputBinaryFile(str)): """Train CNN model on MNIST dataset.""" from tensorflow.python import keras from tensorflow.python.keras import Sequential, backend as K from tensorflow.python.keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense import numpy as np mnistdata = np.load(data) train_x = mnistdata['train_x'] train_y = mnistdata['train_y'] test_x = mnistdata['test_x'] test_y = mnistdata['test_y'] num_classes = 10 img_w = 28 img_h = 28 if K.image_data_format() == 'channels_first': train_x.shape = (-1, 1, img_h, img_w) test_x.shape = (-1, 1, img_h, img_w) input_shape = (1, img_h, img_w) else: train_x.shape = (-1, img_h, img_w, 1) test_x.shape = (-1, img_h, img_w, 1) input_shape = (img_h, img_w, 1) model = Sequential([ Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), Conv2D(64, (3, 3), activation='relu'), MaxPooling2D(pool_size=(2, 2)), Dropout(0.25), Flatten(), Dense(128, activation='relu'), Dropout(0.5), Dense(num_classes, activation='softmax'), ]) model.compile( loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'], ) model.fit( train_x, train_y, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(test_x, test_y), ) score = model.evaluate(test_x, test_y) print('Test loss & accuracy: %s' % (score, )) model.save(model_path)
def write_to_file_path(number_file: OutputBinaryFile(int)): number_file.write(b'42')
def load_task( train_images: str, train_labels: str, test_images: str, test_labels: str, traintest_output: OutputBinaryFile(str), validation_output: OutputBinaryFile(str), ): """Transforms MNIST data from upstream format into numpy array.""" from gzip import GzipFile from pathlib import Path from tensorflow.python.keras.utils import get_file import numpy as np import struct from tensorflow.python.keras.utils import to_categorical def load(path): """Ensures that a file is downloaded locally, then unzips and reads it.""" return GzipFile(get_file(Path(path).name, path)).read() def parse_labels(b: bytes) -> np.array: """Parses numeric labels from input data.""" assert struct.unpack('>i', b[:4])[0] == 0x801 return np.frombuffer(b[8:], dtype=np.uint8) def parse_images(b: bytes) -> np.array: """Parses images from input data.""" assert struct.unpack('>i', b[:4])[0] == 0x803 count = struct.unpack('>i', b[4:8])[0] rows = struct.unpack('>i', b[8:12])[0] cols = struct.unpack('>i', b[12:16])[0] data = np.frombuffer(b[16:], dtype=np.uint8) return data.reshape((count, rows, cols)).astype('float32') / 255 train_x = parse_images(load(train_images)) train_y = to_categorical(parse_labels(load(train_labels))) test_x = parse_images(load(test_images)) test_y = to_categorical(parse_labels(load(test_labels))) # For example purposes, we don't need the entire training set, just enough # to get reasonable accuracy train_x = train_x[:1000, :, :] train_y = train_y[:1000] np.savez_compressed( traintest_output, **{ 'train_x': train_x, 'train_y': train_y, 'test_x': test_x[100:, :, :], 'test_y': test_y[100:], }, ) np.savez_compressed( validation_output, **{ 'val_x': test_x[:100, :, :].reshape(100, 28, 28, 1), 'val_y': test_y[:100] }, )