def test_auto_encoder(tmp_path): print(tmp_path) tensor_shapes = {'label': [28, 28], 'tensor': [28, 28]} json.dump(tensor_shapes, open(tmp_path / 'tensor_shapes.txt', "w")) tensor_types = {'label': 'float', 'tensor': 'float'} json.dump(tensor_types, open(tmp_path / 'tensor_types.txt', "w")) model_config = { 'layers': [ "tf.keras.Sequential([Flatten(),Dense(64, activation='relu'),])", "tf.keras.Sequential([Dense(784, activation='sigmoid'),Reshape((28, 28))], name='output')" ] } user_params = { 'tensor_cols': json.dumps(['tensor']), 'label_col': 'label', 'label_type': 'float', 'batch_size': '128', 'add_output_layer': 'false', 'num_epochs': '1', 'model_config': json.dumps(model_config), 'metrics': json.dumps(['mse', 'mae', 'tf.keras.metrics.RootMeanSquaredError()']), 'optimizer': 'Adam(learning_rate=0.1)', 'loss': 'tf.keras.losses.BinaryCrossentropy()', } args: TrainTaskConfig = TrainTaskConfig( dataset_file='autoencoder.tfrecords', tf_context=None, num_workers=1, cluster=None, task_type='chief', task_index=0, work_dir=str(tmp_path), dataset=None, dataset_length=100, saved_model_dir=str(tmp_path / 'saved_model_dir'), user_params=user_params) keras_sequential_main.main(args)
def test_keras_sequential_multi_classification_one_worker(tmp_path): print(tmp_path) tensor_shapes = { 'tensor': [784], } json.dump(tensor_shapes, open(tmp_path / 'tensor_shapes.txt', "w")) tensor_types = {'tensor': 'double'} json.dump(tensor_types, open(tmp_path / 'tensor_types.txt', "w")) with open(tmp_path / 'bc_data_1', 'w') as f: f.write("10") model_config = { 'layers': [ "Reshape((28, 28, 1))", "Conv2D(5, 7)", "Flatten()", "Dense(1024, activation='relu')", "Dense(512, activation='relu')", ] } user_params = { 'tensor_cols': json.dumps(['tensor']), 'label_col': 'label', 'label_type': 'double', 'batch_size': '32', 'num_epochs': '1', 'model_config': json.dumps(model_config), 'ALINK:ONLY_ONE_WORKER': 'true', 'validation_split': '0.2', 'save_checkpoints_epochs': '1.', 'save_best_only': 'true', 'best_exporter_metric': 'msle', 'ALINK:bc_1': str(tmp_path / 'bc_data_1') } args: TrainTaskConfig = TrainTaskConfig( dataset_file='multi_classification_dataset.tfrecords', tf_context=None, num_workers=1, cluster=None, task_type='chief', task_index=0, work_dir=str(tmp_path), dataset=None, dataset_length=8000, saved_model_dir=str(tmp_path / 'saved_model_dir'), user_params=user_params) keras_sequential_main.main(args)
def test_keras_sequential_regression_one_worker(tmp_path): print(tmp_path) tensor_shapes = { 'tensor': [784], } json.dump(tensor_shapes, open(tmp_path / 'tensor_shapes.txt', "w")) tensor_types = {'tensor': 'double'} json.dump(tensor_types, open(tmp_path / 'tensor_types.txt', "w")) model_config = { 'layers': [ "Dense(1024, activation='relu')", "Dense(512, activation='relu')", ] } user_params = { 'tensor_cols': json.dumps(['tensor']), 'label_col': 'label', 'label_type': 'float', 'batch_size': '32', 'num_epochs': '10', 'ALINK:ONLY_ONE_WORKER': 'true', 'validation_split': '0.2', 'save_checkpoints_epochs': '0.2', 'save_best_only': 'true', 'best_exporter_metric': 'mean_squared_logarithmic_error', 'model_config': json.dumps(model_config), 'optimizer': 'Adam(learning_rate=0.1)' } args: TrainTaskConfig = TrainTaskConfig( dataset_file='regression_dataset.tfrecords', tf_context=None, num_workers=1, cluster=None, task_type='chief', task_index=0, work_dir=str(tmp_path), dataset=None, dataset_length=8000, saved_model_dir=str(tmp_path / 'saved_model_dir'), user_params=user_params) keras_sequential_main.main(args)
def test_keras_sequential_classification_string(tmp_path): print(tmp_path) tensor_shapes = { 'tensor': [], } json.dump(tensor_shapes, open(tmp_path / 'tensor_shapes.txt', "w")) tensor_types = {'tensor': 'string'} json.dump(tensor_types, open(tmp_path / 'tensor_types.txt', "w")) with open(tmp_path / 'bc_data_1', 'w') as f: f.write("2") model_config = { 'layers': [ "hub.KerasLayer('https://tfhub.dev/google/nnlm-de-dim50/2', input_shape=[1], dtype=tf.string)", "Flatten()", ] } user_params = { 'tensor_cols': json.dumps(['tensor']), 'label_col': 'label', 'label_type': 'float', 'batch_size': '32', 'num_epochs': '2', 'model_config': json.dumps(model_config), 'optimizer': 'Adam(learning_rate=0.1)' } args: TrainTaskConfig = TrainTaskConfig( dataset_file='binary_classification_string_data.tfrecords', tf_context=None, num_workers=1, cluster=None, task_type='chief', task_index=0, work_dir=str(tmp_path), dataset=None, dataset_length=100, saved_model_dir=str(tmp_path / 'saved_model_dir'), user_params=user_params) keras_sequential_main.main(args)
def test_keras_sequential_binary_classification(tmp_path): print(tmp_path) tensor_shapes = { 'tensor': [784], } json.dump(tensor_shapes, open(tmp_path / 'tensor_shapes.txt', "w")) tensor_types = {'tensor': 'double'} json.dump(tensor_types, open(tmp_path / 'tensor_types.txt', "w")) with open(tmp_path / 'bc_data_1', 'w') as f: f.write("2") model_config = { 'layers': [ "Dense(1024, activation='relu')", "Dense(512, activation='relu')", ] } user_params = { 'tensor_cols': json.dumps(['tensor']), 'label_col': 'label', 'label_type': 'float', 'batch_size': '32', 'num_epochs': '2', 'model_config': json.dumps(model_config), 'ALINK:bc_1': str(tmp_path / 'bc_data_1') } args: TrainTaskConfig = TrainTaskConfig( dataset_file='binary_classification_dataset.tfrecords', tf_context=None, num_workers=1, cluster=None, task_type='chief', task_index=0, work_dir=str(tmp_path), dataset=None, dataset_length=100, saved_model_dir=str(tmp_path / 'saved_model_dir'), user_params=user_params) keras_sequential_main.main(args)
def test_keras_sequential_regression(tmp_path): print(tmp_path) tensor_shapes = { 'tensor': [784], } json.dump(tensor_shapes, open(tmp_path / 'tensor_shapes.txt', "w")) tensor_types = {'tensor': 'double'} json.dump(tensor_types, open(tmp_path / 'tensor_types.txt', "w")) model_config = { 'layers': [ "Dense(1024, activation='relu')", "Dense(512, activation='relu')", ] } user_params = { 'tensor_cols': json.dumps(['tensor']), 'label_col': 'label', 'label_type': 'float', 'batch_size': '32', 'num_epochs': '2', 'model_config': json.dumps(model_config), 'optimizer': 'Adam(learning_rate=0.1)' } args: TrainTaskConfig = TrainTaskConfig( dataset_file='regression_dataset.tfrecords', tf_context=None, num_workers=1, cluster=None, task_type='chief', task_index=0, work_dir=str(tmp_path), dataset=None, dataset_length=100, saved_model_dir=str(tmp_path / 'saved_model_dir'), user_params=user_params) keras_sequential_main.main(args)
def main(task_config: TrainTaskConfig): keras_sequential_main.main(task_config)