def build_network(self): """ Define the FastEstimator network flow. Args: None Returns: network: KerasNetwork object """ epsilon = 0.04 network = fe.Network(ops=[ Watch(inputs="x"), ModelOp(model=self.model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="base_ce"), FGSM( data="x", loss="base_ce", outputs="x_adverse", epsilon=epsilon), ModelOp(model=self.model, inputs="x_adverse", outputs="y_pred_adv"), CrossEntropy(inputs=("y_pred_adv", "y"), outputs="adv_ce"), Average(inputs=("base_ce", "adv_ce"), outputs="avg_ce"), UpdateOp(model=self.model, loss_name="avg_ce") ]) return network
def get_estimator(epochs=10, batch_size=50, epsilon=0.04, save_dir=tempfile.mkdtemp()): train_data, eval_data = cifar10.load_data() test_data = eval_data.split(0.5) pipeline = fe.Pipeline( train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=batch_size, ops=[Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616))]) model = fe.build(model_fn=lambda: LeNet(input_shape=(32, 32, 3)), optimizer_fn="adam", model_name="adv_model") network = fe.Network(ops=[ Watch(inputs="x"), ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="base_ce"), FGSM(data="x", loss="base_ce", outputs="x_adverse", epsilon=epsilon), ModelOp(model=model, inputs="x_adverse", outputs="y_pred_adv"), CrossEntropy(inputs=("y_pred_adv", "y"), outputs="adv_ce"), Average(inputs=("base_ce", "adv_ce"), outputs="avg_ce"), UpdateOp(model=model, loss_name="avg_ce") ]) traces = [ Accuracy(true_key="y", pred_key="y_pred", output_name="clean_accuracy"), Accuracy(true_key="y", pred_key="y_pred_adv", output_name="adversarial_accuracy"), BestModelSaver(model=model, save_dir=save_dir, metric="base_ce", save_best_mode="min"), ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, monitor_names=["base_ce", "adv_ce"], log_steps=1000) return estimator
def test_categorical_crossentropy_weights(self): ce = CrossEntropy(inputs='x', outputs='x', class_weights=self.tf_cat_weights) ce.build('tf') output = ce.forward(data=[self.tf_pred_cat, self.tf_true_cat], state={}) self.assertTrue(np.allclose(output.numpy(), 0.54055756))
def test_sparse_categorical_crossentropy_weights(self): ce = CrossEntropy(inputs='x', outputs='x', class_weights=self.tf_sparse_weights) ce.build('tf') output = ce.forward(data=[self.tf_pred_sparse, self.tf_true_sparse], state={}) self.assertTrue(np.allclose(output.numpy(), 3.532212))
def test_torch_input_weights(self): ce = CrossEntropy(inputs='x', outputs='x', class_weights=self.torch_binary_weights) ce.build('torch') output = ce.forward( data=[self.torch_pred_binary, self.torch_true_binary], state={}) self.assertTrue(np.allclose(output.detach().numpy(), 0.2797609))
def test_binary_crossentropy_weights(self): ce = CrossEntropy(inputs='x', outputs='x', class_weights=self.tf_binary_weights) ce.build('tf') output = ce.forward(data=[self.tf_pred_binary, self.tf_true_binary], state={}) self.assertTrue(np.allclose(output.numpy(), -56.221874))
def test_pytorch_weight_decay_vs_l2(self): # Get Data train_data, _ = mnist.load_data() t_d = train_data.split(128) # Initializing models pytorch_wd = fe.build(model_fn=MyNet_torch, optimizer_fn=lambda x: torch.optim.SGD(params=x, lr=0.01, weight_decay=self.beta)) pytorch_l2 = fe.build(model_fn=MyNet_torch, optimizer_fn=lambda x: torch.optim.SGD(params=x, lr=0.01)) # Initialize pipeline pipeline = fe.Pipeline(train_data=t_d, batch_size=128, ops=[ExpandDims(inputs="x", outputs="x", axis=0), Minmax(inputs="x", outputs="x")]) # Define the two pytorch networks network_weight_decay = fe.Network(ops=[ ModelOp(model=pytorch_wd, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=pytorch_wd, loss_name="ce") ]) network_l2 = fe.Network(ops=[ ModelOp(model=pytorch_l2, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), L2Regularizaton(inputs="ce", outputs="l2", model=pytorch_l2, beta=self.beta), UpdateOp(model=pytorch_l2, loss_name="l2") ]) # defining traces traces = [Accuracy(true_key="y", pred_key="y_pred")] # Setting up estimators estimator_wd = fe.Estimator(pipeline=pipeline, network=network_weight_decay, epochs=1, traces=traces, train_steps_per_epoch=1) estimator_l2 = fe.Estimator(pipeline=pipeline, network=network_l2, epochs=1, traces=traces, train_steps_per_epoch=1) # Training print('********************************Pytorch weight decay training************************************') estimator_wd.fit() print() print('********************************Pytorch L2 Regularization training************************************') estimator_l2.fit() # testing weights count = 0 for wt, l2 in zip(pytorch_wd.parameters(), pytorch_l2.parameters()): if ((wt - l2).abs()).sum() < torch.tensor(10**-6): count += 1 self.assertTrue(count == 6)
def get_estimator(epochs=24, batch_size=128, lr_epochs=100, max_train_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): # step 1: prepare dataset train_data, test_data = load_data() pipeline = fe.Pipeline( train_data=train_data, eval_data=test_data, batch_size=batch_size, ops=[ Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616)), PadIfNeeded(min_height=40, min_width=40, image_in="x", image_out="x", mode="train"), RandomCrop(32, 32, image_in="x", image_out="x", mode="train"), Sometimes(HorizontalFlip(image_in="x", image_out="x", mode="train")), CoarseDropout(inputs="x", outputs="x", mode="train", max_holes=1), ChannelTranspose(inputs="x", outputs="x"), Onehot(inputs="y", outputs="y", mode="train", num_classes=10, label_smoothing=0.2) ]) # step 2: prepare network model = fe.build(model_fn=ResNet9, optimizer_fn="sgd") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) # get the max learning rate lr_max = search_max_lr(pipeline=pipeline, model=model, network=network, epochs=lr_epochs) lr_min = lr_max / 40 print(f"The maximum LR: {lr_max}, and minimun LR: {lr_min}") mid_step = int(epochs * 0.45 * len(train_data) / batch_size) end_step = int(epochs * len(train_data) / batch_size) # reinitialize the model model = fe.build(model_fn=ResNet9, optimizer_fn="sgd") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) # step 3: prepare estimator traces = [ Accuracy(true_key="y", pred_key="y_pred"), BestModelSaver(model=model, save_dir=save_dir, metric="accuracy", save_best_mode="max"), LRScheduler(model=model, lr_fn=lambda step: super_schedule(step, lr_max, lr_min, mid_step, end_step)) ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch) return estimator
def finetune_model(model, epochs, batch_size, max_train_steps_per_epoch, save_dir): train_data, test_data = load_data() train_data = train_data.split(0.1) pipeline = fe.Pipeline(train_data=train_data, eval_data=test_data, batch_size=batch_size, ops=[ ToFloat(inputs="x", outputs="x"), ]) network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=["y_pred", "y"], outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) traces = [ Accuracy(true_key="y", pred_key="y_pred"), ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch) estimator.fit()
def get_estimator(epochs=2, batch_size=32): # step 1 train_data, eval_data = mnist.load_data() pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, batch_size=batch_size, ops=[ ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x") ]) # step 2 model = fe.build(model_fn=LeNet, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs=["y_pred", "feature_vector"], intermediate_layers='dense'), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), CustomLoss(inputs=("feature_vector", "feature_selected"), outputs="feature_loss"), LambdaOp(fn=lambda x, y: x + y, inputs=("ce", "feature_loss"), outputs="total_loss"), UpdateOp(model=model, loss_name="total_loss") ]) # step 3 traces = [ MemoryBank(inputs=("feature_vector", "y"), outputs="feature_selected") ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces) return estimator
def get_estimator(epochs=10, batch_size=32, train_steps_per_epoch=None): train_data, eval_data = cifair10.load_data() pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, batch_size=batch_size, ops=[Normalize(inputs="x", outputs="x")]) # step 2 model = fe.build(model_fn=lambda: LeNet(input_shape=(32, 32, 3)), optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) # step 3 traces = [ Accuracy(true_key="y", pred_key="y_pred"), ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, train_steps_per_epoch=train_steps_per_epoch) return estimator
def test_network_transform_lenet_torch(self): model = fe.build( model_fn=LeNetTorch, optimizer_fn=lambda x: torch.optim.Adam(params=x, lr=1.0)) weight = get_torch_lenet_model_weight(model) network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) batch = { "x": np.ones((1, 1, 28, 28), dtype=np.float32), "y": np.array([[1]], dtype=np.float32) } batch = network.transform(data=batch, mode="train") with self.subTest("output y_pred check"): self.assertTrue("y_pred" in batch.keys()) self.assertIsNotNone(batch["y_pred"]) with self.subTest("output ce check"): self.assertTrue("ce" in batch.keys()) self.assertIsNotNone(batch["ce"]) with self.subTest("check whether model weight changed"): weight2 = get_torch_lenet_model_weight(model) self.assertFalse(is_equal(weight, weight2))
def run_test(mixed_precision, merge_grad, gradient): lr = 0.1 pipeline = fe.Pipeline(train_data=self.train_data, batch_size=4, ops=[ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x")]) model = fe.build(model_fn=LeNet_tf, optimizer_fn=lambda: tf.optimizers.SGD(lr), mixed_precision=mixed_precision) network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), GradientOp(model=model, finals="ce", outputs="grad"), UpdateOp(model=model, loss_name="ce", gradients=gradient, merge_grad=merge_grad), ]) traces = [ CheckNetworkWeight(model=model, grad_key="grad", merge_grad=merge_grad, test_self=self, lrs=lr, framework="tf") ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2, traces=traces, train_steps_per_epoch=2) estimator.fit(warmup=False)
def get_estimator(max_words=10000, max_len=500, epochs=10, batch_size=64, train_steps_per_epoch=None, eval_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): # step 1. prepare data train_data, eval_data = imdb_review.load_data(max_len, max_words) pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, batch_size=batch_size, ops=Reshape(1, inputs="y", outputs="y")) # step 2. prepare model model = fe.build(model_fn=lambda: ReviewSentiment(max_words=max_words), optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="loss"), UpdateOp(model=model, loss_name="loss") ]) # step 3.prepare estimator traces = [Accuracy(true_key="y", pred_key="y_pred"), BestModelSaver(model=model, save_dir=save_dir)] estimator = fe.Estimator(network=network, pipeline=pipeline, epochs=epochs, traces=traces, train_steps_per_epoch=train_steps_per_epoch, eval_steps_per_epoch=eval_steps_per_epoch) return estimator
def get_estimator(epochs=10, batch_size=32, extend_ds=False): train_data, eval_data = cifair10.load_data() if extend_ds: train_data = ExtendDataset(dataset=train_data, spoof_length=len(train_data) * 2) epochs //= 2 pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, batch_size=batch_size, ops=[Normalize(inputs="x", outputs="x")]) # step 2 model = fe.build(model_fn=lambda: LeNet(input_shape=(32, 32, 3)), optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) # step 3 traces = [ Accuracy(true_key="y", pred_key="y_pred"), ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces) return estimator
def _build_estimator(model: Union[tf.keras.Model, torch.nn.Module], trace: Traceability, axis: int = -1): train_data, eval_data = mnist.load_data() test_data = eval_data.split(0.5) batch_size = 32 pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=batch_size, ops=[ExpandDims(inputs="x", outputs="x", axis=axis), Minmax(inputs="x", outputs="x")]) network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) traces = [ Accuracy(true_key="y", pred_key="y_pred"), LRScheduler(model=model, lr_fn=lambda step: cosine_decay(step, cycle_length=3750, init_lr=1e-3)), trace ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=1, traces=traces, max_train_steps_per_epoch=1, max_eval_steps_per_epoch=None) fake_data = tf.ones(shape=(batch_size, 28, 28, 1)) if axis == -1 else torch.ones(size=(batch_size, 1, 28, 28)) model.fe_input_spec = FeInputSpec(fake_data, model) return estimator
def get_estimator(epochs=2, batch_size=32, save_dir=tempfile.mkdtemp()): # step 1 train_data, eval_data = mnist.load_data() test_data = eval_data.split(0.5) pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=batch_size, ops=[ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x")]) # step 2 model = fe.build(model_fn=LeNet, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) # step 3 traces = [ Accuracy(true_key="y", pred_key="y_pred"), BestModelSaver(model=model, save_dir=save_dir, metric="accuracy", save_best_mode="max"), LRScheduler(model=model, lr_fn=lambda step: cosine_decay(step, cycle_length=3750, init_lr=1e-3)) ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces) return estimator
def test_mode_ds_id_interaction(self): train_data, eval_data = mnist.load_data() test_data = eval_data.split(0.5) pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=32, ops=[ ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x") ]) model = fe.build(model_fn=LeNet, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce", ds_id="ds_1") ]) pipeline_data = pipeline.transform(data=train_data[0], mode="train") data1 = network.transform(data=pipeline_data, mode="infer", ds_id="ds_1") assert "ce" not in data1 data2 = network.transform(data=pipeline_data, mode="infer", ds_id="ds_2") assert "ce" not in data2
def run_test(mixed_precision, merge_grad, gradient): lr = 0.1 lr2 = 0.01 pipeline = fe.Pipeline(train_data=self.train_data, batch_size=4, ops=[ExpandDims(inputs="x", outputs="x", axis=0), Minmax(inputs="x", outputs="x")]) optimizer_fn = RepeatScheduler( [lambda x: torch.optim.SGD(params=x, lr=lr), lambda x: torch.optim.SGD(params=x, lr=lr2)]) model = fe.build(model_fn=LeNet_torch, optimizer_fn=optimizer_fn, mixed_precision=mixed_precision) network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), GradientOp(model=model, finals="ce", outputs="grad"), UpdateOp(model=model, loss_name="ce", gradients=gradient, merge_grad=merge_grad), ]) traces = [ CheckNetworkWeight(model=model, grad_key="grad", merge_grad=merge_grad, test_self=self, framework="torch", lrs=[lr, lr2, lr, lr2], work_intervals=[[1, 2], [2, 3], [3, 4], [4, 5]]) ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=4, traces=traces, train_steps_per_epoch=2) estimator.fit(warmup=False)
def test_estimator_configure_loader_torch_data_loader_tf_model(self): loader = get_sample_torch_dataloader() pipeline = fe.Pipeline(train_data=loader) model = fe.build(model_fn=LeNetTf, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x_out", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) est = fe.Estimator(pipeline=pipeline, network=network, max_train_steps_per_epoch=3, epochs=1) est.system.mode = "train" new_loader = est._configure_loader(loader) with self.subTest("check loader type"): strategy = tf.distribute.get_strategy() if isinstance(strategy, tf.distribute.MirroredStrategy): self.assertIsInstance(new_loader, tf.distribute.DistributedDataset) else: self.assertIsInstance(new_loader, tf.data.Dataset) with self.subTest("max_train_steps_per_epoch=3"): iterator = iter(new_loader) for i in range(3): batch = next(iterator) with self.assertRaises(StopIteration): batch = next(iterator)
def test_tf_static_superloss_categorical_ce(self): sl = SuperLoss(CrossEntropy(inputs=['y_pred', 'y'], outputs='ce')) sl.build(framework="tf", device=None) output = self.do_forward(sl, data=[self.tf_pred_cat, self.tf_true_cat], state=self.state) self.assertTrue(np.allclose(output.numpy(), -0.0026386082))
def instantiate_system(): system = sample_system_object() model = fe.build(model_fn=fe.architecture.pytorch.LeNet, optimizer_fn='adam', model_name='tf') system.network = fe.Network(ops=[ ModelOp(model=model, inputs="x_out", outputs="y_pred"), SuperLoss(CrossEntropy(inputs=['y_pred', 'y'], outputs='ce')) ]) return system
def test_torch_superloss_binary_ce(self): sl = SuperLoss(CrossEntropy(inputs=['y_pred', 'y'], outputs='ce')) sl.build(framework="torch", device="cuda:0" if torch.cuda.is_available() else "cpu") output = sl.forward( data=[self.torch_pred_binary, self.torch_true_binary], state=self.state) self.assertTrue( np.allclose(output.detach().to("cpu").numpy(), -0.0026238672))
def get_estimator(epsilon=0.04, epochs=20, batch_size=32, code_length=16, train_steps_per_epoch=None, eval_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): # step 1 train_data, eval_data = cifair10.load_data() test_data = eval_data.split(0.5) pipeline = fe.Pipeline( train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=batch_size, ops=[Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616))]) # step 2 model = fe.build(model_fn=lambda: ecc_lenet(code_length=code_length), optimizer_fn="adam") network = fe.Network(ops=[ Watch(inputs="x", mode=('eval', 'test')), ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="base_ce"), UpdateOp(model=model, loss_name="base_ce"), FGSM(data="x", loss="base_ce", outputs="x_adverse", epsilon=epsilon, mode=('eval', 'test')), ModelOp(model=model, inputs="x_adverse", outputs="y_pred_adv", mode=('eval', 'test')), CrossEntropy(inputs=("y_pred_adv", "y"), outputs="adv_ce", mode=('eval', 'test')), Average(inputs=("base_ce", "adv_ce"), outputs="avg_ce", mode='eval') ]) # step 3 traces = [ Accuracy(true_key="y", pred_key="y_pred", output_name="base_accuracy"), Accuracy(true_key="y", pred_key="y_pred_adv", output_name="adversarial_accuracy"), BestModelSaver(model=model, save_dir=save_dir, metric="avg_ce", save_best_mode="min", load_best_final=True) ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, train_steps_per_epoch=train_steps_per_epoch, eval_steps_per_epoch=eval_steps_per_epoch, monitor_names=["adv_ce", "avg_ce"]) return estimator
def get_estimator(epochs=2, batch_size=32, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): # step 1 train_data, eval_data = mnist.load_data() test_data = eval_data.split(0.5) pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=batch_size, ops=[ ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x") ], num_process=0) # step 2 model = fe.build(model_fn=LeNet, optimizer_fn="adam") print([f"{idx}: {x.name}" for idx, x in enumerate(model.submodules)]) network = fe.Network(ops=[ Watch(inputs="x"), ModelOp(model=model, inputs="x", outputs=["y_pred", "embedding"], intermediate_layers='dense'), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), GradientOp(finals="embedding", inputs="x", outputs="grads"), UpdateOp(model=model, loss_name="ce") ]) # step 3 traces = [ Accuracy(true_key="y", pred_key="y_pred"), Inspector(), BestModelSaver(model=model, save_dir=save_dir, metric="accuracy", save_best_mode="max"), LRScheduler(model=model, lr_fn=lambda step: cosine_decay( step, cycle_length=3750, init_lr=1e-3)), TensorBoard(log_dir="tf_logs", write_embeddings="embedding", embedding_labels="y") ] estimator = fe.Estimator( pipeline=pipeline, network=network, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch) return estimator
def get_estimator(epochs=50, batch_size=128, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): # step 1 train_data, eval_data = cifair100.load_data() # Add label noise to simulate real-world labeling problems corrupt_dataset(train_data) test_data = eval_data.split(range(len(eval_data) // 2)) pipeline = fe.Pipeline( train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=batch_size, ops=[ Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616)), PadIfNeeded(min_height=40, min_width=40, image_in="x", image_out="x", mode="train"), RandomCrop(32, 32, image_in="x", image_out="x", mode="train"), Sometimes(HorizontalFlip(image_in="x", image_out="x", mode="train")), CoarseDropout(inputs="x", outputs="x", mode="train", max_holes=1), ChannelTranspose(inputs="x", outputs="x") ]) # step 2 model = fe.build(model_fn=big_lenet, optimizer_fn='adam') network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), SuperLoss(CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), output_confidence="confidence"), UpdateOp(model=model, loss_name="ce") ]) # step 3 traces = [ MCC(true_key="y", pred_key="y_pred"), BestModelSaver(model=model, save_dir=save_dir, metric="mcc", save_best_mode="max", load_best_final=True), LabelTracker(metric="confidence", label="data_labels", label_mapping={ "Normal": 0, "Corrupted": 1 }, mode="train", outputs="label_confidence"), ImageSaver(inputs="label_confidence", save_dir=save_dir, mode="train"), ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch) return estimator
def setUpClass(cls): train_data, eval_data = mnist.load_data() cls.pipeline = fe.Pipeline(train_data=train_data) model = fe.build(model_fn=LeNetTf, optimizer_fn="adam") cls.network = fe.Network(ops=[ ModelOp(model=model, inputs="x_out", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ])
def get_estimator(epochs=2, batch_size=32, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): # step 1. prepare data train_data, eval_data = breast_cancer.load_data() # Apply some global pre-processing to the data scaler = StandardScaler() train_data["x"] = scaler.fit_transform(train_data["x"]) eval_data["x"] = scaler.transform(eval_data["x"]) pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, batch_size=batch_size) # step 2. prepare model model = fe.build(model_fn=DNN, optimizer_fn="adam") class_weights = {0: 2.0, 1: 2.0} network = fe.Network(ops=[ ModelOp(inputs="x", model=model, outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce_weighted", class_weights=class_weights), UpdateOp(model=model, loss_name="ce") ]) # step 3.prepare estimator traces = [ Accuracy(true_key="y", pred_key="y_pred"), BestModelSaver(model=model, save_dir=save_dir, metric="accuracy", save_best_mode="max") ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, log_steps=10, traces=traces, monitor_names="ce_weighted", max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch) return estimator
def get_estimator(): # step 1 pipeline = create_pipeline() # step 2 model = fe.build(model_fn=LeNet, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) # step 3 estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2) return estimator
def get_estimator(max_len=20, epochs=10, batch_size=64, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None, pretrained_model='bert-base-uncased', save_dir=tempfile.mkdtemp(), data_dir=None): # step 1 prepare data train_data, eval_data, data_vocab, label_vocab = german_ner.load_data(root_dir=data_dir) tokenizer = BertTokenizer.from_pretrained(pretrained_model, do_lower_case=True) tag2idx = char2idx(label_vocab) pipeline = fe.Pipeline( train_data=train_data, eval_data=eval_data, batch_size=batch_size, ops=[ Tokenize(inputs="x", outputs="x", tokenize_fn=tokenizer.tokenize), WordtoId(inputs="x", outputs="x", mapping=tokenizer.convert_tokens_to_ids), WordtoId(inputs="y", outputs="y", mapping=tag2idx), PadSequence(max_len=max_len, inputs="x", outputs="x"), PadSequence(max_len=max_len, value=len(tag2idx), inputs="y", outputs="y"), AttentionMask(inputs="x", outputs="x_masks") ]) # step 2. prepare model bert_config = BertConfig.from_pretrained(pretrained_model) num_hidden_layers = bert_config.to_dict()['num_hidden_layers'] head_masks = [None] * num_hidden_layers model = fe.build(model_fn=lambda: NERModel(head_masks=head_masks, pretrained_model=pretrained_model), optimizer_fn=lambda x: torch.optim.Adam(x, lr=1e-5)) network = fe.Network(ops=[ ModelOp(model=model, inputs=["x", "x_masks"], outputs="y_pred"), Reshape(inputs="y", outputs="y", shape=(-1, )), Reshape(inputs="y_pred", outputs="y_pred", shape=(-1, 24)), CrossEntropy(inputs=("y_pred", "y"), outputs="loss"), UpdateOp(model=model, loss_name="loss") ]) traces = [Accuracy(true_key="y", pred_key="y_pred"), BestModelSaver(model=model, save_dir=save_dir)] # step 3 prepare estimator estimator = fe.Estimator(network=network, pipeline=pipeline, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch) return estimator