def get_estimator(epsilon=0.04, epochs=10, batch_size=32, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): # step 1 train_data, eval_data = cifair10.load_data() test_data = eval_data.split(0.5) pipeline = fe.Pipeline( train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=batch_size, ops=[Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616))]) # step 2 model = fe.build(model_fn=lambda: LeNet(input_shape=(32, 32, 3)), optimizer_fn="adam") network = fe.Network(ops=[ Watch(inputs="x"), ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="base_ce"), FGSM(data="x", loss="base_ce", outputs="x_adverse", epsilon=epsilon), ModelOp(model=model, inputs="x_adverse", outputs="y_pred_adv"), CrossEntropy(inputs=("y_pred_adv", "y"), outputs="adv_ce"), Average(inputs=("base_ce", "adv_ce"), outputs="avg_ce"), UpdateOp(model=model, loss_name="avg_ce") ]) # step 3 traces = [ Accuracy(true_key="y", pred_key="y_pred", output_name="base accuracy"), Accuracy(true_key="y", pred_key="y_pred_adv", output_name="adversarial accuracy"), BestModelSaver(model=model, save_dir=save_dir, metric="adv_ce", save_best_mode="min"), ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch, monitor_names=["base_ce", "adv_ce"]) return estimator
def get_estimator(batch_size=4, epochs=2, max_train_steps_per_epoch=None, log_steps=100, style_weight=5.0, content_weight=1.0, tv_weight=1e-4, save_dir=tempfile.mkdtemp(), style_img_path='Vassily_Kandinsky,_1913_-_Composition_7.jpg', data_dir=None): train_data, _ = mscoco.load_data(root_dir=data_dir, load_bboxes=False, load_masks=False, load_captions=False) device = "cuda" if torch.cuda.is_available() else "cpu" style_img = cv2.imread(style_img_path) assert style_img is not None, "cannot load the style image, please go to the folder with style image" style_img = cv2.resize(style_img, (256, 256)) style_img = (style_img.astype(np.float32) - 127.5) / 127.5 pipeline = fe.Pipeline( train_data=train_data, batch_size=batch_size, ops=[ ReadImage(inputs="image", outputs="image"), Normalize(inputs="image", outputs="image", mean=1.0, std=1.0, max_pixel_value=127.5), Resize(height=256, width=256, image_in="image", image_out="image"), LambdaOp(fn=lambda: style_img, outputs="style_image"), ChannelTranspose(inputs=["image", "style_image"], outputs=["image", "style_image"]) ]) model = fe.build(model_fn=StyleTransferNet, model_name="style_transfer_net", optimizer_fn=lambda x: torch.optim.Adam(x, lr=1e-3)) network = fe.Network(ops=[ ModelOp(inputs="image", model=model, outputs="image_out"), ExtractVGGFeatures(inputs="style_image", outputs="y_style", device=device), ExtractVGGFeatures(inputs="image", outputs="y_content", device=device), ExtractVGGFeatures(inputs="image_out", outputs="y_pred", device=device), StyleContentLoss(style_weight=style_weight, content_weight=content_weight, tv_weight=tv_weight, inputs=('y_pred', 'y_style', 'y_content', 'image_out'), outputs='loss'), UpdateOp(model=model, loss_name="loss") ]) estimator = fe.Estimator(network=network, pipeline=pipeline, traces=ModelSaver(model=model, save_dir=save_dir, frequency=1), epochs=epochs, max_train_steps_per_epoch=max_train_steps_per_epoch, log_steps=log_steps) return estimator
def get_estimator(epochs=2, batch_size=32, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): # step 1 train_data, eval_data = mnist.load_data() test_data = eval_data.split(0.5) pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=batch_size, ops=[ ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x") ], num_process=0) # step 2 model = fe.build(model_fn=LeNet, optimizer_fn="adam") print([f"{idx}: {x.name}" for idx, x in enumerate(model.submodules)]) network = fe.Network(ops=[ Watch(inputs="x"), ModelOp(model=model, inputs="x", outputs=["y_pred", "embedding"], intermediate_layers='dense'), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), GradientOp(finals="embedding", inputs="x", outputs="grads"), UpdateOp(model=model, loss_name="ce") ]) # step 3 traces = [ Accuracy(true_key="y", pred_key="y_pred"), Inspector(), BestModelSaver(model=model, save_dir=save_dir, metric="accuracy", save_best_mode="max"), LRScheduler(model=model, lr_fn=lambda step: cosine_decay( step, cycle_length=3750, init_lr=1e-3)), TensorBoard(log_dir="tf_logs", write_embeddings="embedding", embedding_labels="y") ] estimator = fe.Estimator( pipeline=pipeline, network=network, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch) return estimator
def test_estimator_warmup_trace_missing_key(self): loader = get_sample_tf_dataset() pipeline = fe.Pipeline(train_data=loader) # "x", "y" model = fe.build(model_fn=LeNetTf, optimizer_fn="adam") network = fe.Network(ops=[ EpochScheduler( { 1: ModelOp(model=model, inputs="x", outputs="y_pred"), 2: ModelOp(model=model, inputs="x", outputs="y_pred") }) # miss key x_out ]) est = fe.Estimator(pipeline=pipeline, network=network, epochs=2, traces=[Trace(inputs="z")]) # miss key "z" est._prepare_traces(run_modes={"train", "eval"}) with self.assertRaises(AssertionError): est._warmup()
def test_estimator_warmup_torch_dataset_tf_model_smoke(self): loader = get_sample_torch_dataloader(expand_axis=-1) pipeline = fe.Pipeline(train_data=loader) # "x", "y" model = fe.build(model_fn=LeNetTf, optimizer_fn="adam") network = fe.Network(ops=[ModelOp(model=model, inputs="x", outputs="y_pred")]) est = fe.Estimator(pipeline=pipeline, network=network, epochs=1, traces=[Trace(inputs="y_pred")]) est._prepare_traces(run_modes={"train", "eval"}) est._warmup() self.assertTrue(True)
def setUpClass(cls): train_data, eval_data = mnist.load_data() cls.pipeline = fe.Pipeline(train_data=train_data) model = fe.build(model_fn=LeNetTf, optimizer_fn="adam") cls.network = fe.Network(ops=[ ModelOp(model=model, inputs="x_out", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ])
def get_estimator(epochs=50, batch_size=128, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): # step 1 train_data, eval_data = cifair100.load_data() # Add label noise to simulate real-world labeling problems corrupt_dataset(train_data) test_data = eval_data.split(range(len(eval_data) // 2)) pipeline = fe.Pipeline( train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=batch_size, ops=[ Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616)), PadIfNeeded(min_height=40, min_width=40, image_in="x", image_out="x", mode="train"), RandomCrop(32, 32, image_in="x", image_out="x", mode="train"), Sometimes(HorizontalFlip(image_in="x", image_out="x", mode="train")), CoarseDropout(inputs="x", outputs="x", mode="train", max_holes=1), ChannelTranspose(inputs="x", outputs="x") ]) # step 2 model = fe.build(model_fn=big_lenet, optimizer_fn='adam') network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), SuperLoss(CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), output_confidence="confidence"), UpdateOp(model=model, loss_name="ce") ]) # step 3 traces = [ MCC(true_key="y", pred_key="y_pred"), BestModelSaver(model=model, save_dir=save_dir, metric="mcc", save_best_mode="max", load_best_final=True), LabelTracker(metric="confidence", label="data_labels", label_mapping={ "Normal": 0, "Corrupted": 1 }, mode="train", outputs="label_confidence"), ImageSaver(inputs="label_confidence", save_dir=save_dir, mode="train"), ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch) return estimator
def instantiate_system(): system = sample_system_object_torch() model = fe.build(model_fn=fe.architecture.pytorch.LeNet, optimizer_fn='adam', model_name='torch') system.network = fe.Network(ops=[ fe.op.tensorop.meta.Repeat(op=TestTensorOp( inputs="x_out", outputs="x_out", mode="train", var=1), repeat=2), ModelOp(model=model, inputs="x_out", outputs="y_pred") ]) return system
def test_estimator_check_network_op_trace_invoke_sequence_tf_backend(self): epochs = 1 batches = 10 # dataset has 100 sample, and batch_size is 10 iostream = StringIO() loader = get_sample_tf_dataset() pipeline = fe.Pipeline(test_data=loader) model = fe.build(model_fn=LeNetTf, optimizer_fn="adam") ops = [ ModelOp(model=model, inputs="x", outputs="y_pred"), ShoutNameOp(name="A", iostream=iostream), ShoutNameOp(name="B", iostream=iostream) ] network = fe.Network(ops=ops) traces = [ShoutNameTrace(name="a", iostream=iostream), ShoutNameTrace(name="b", iostream=iostream)] est = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces) est.test() # create the expected calling sequence in another iostream iostream2 = StringIO() ops2 = [ShoutNameOp(name="A", iostream=iostream2), ShoutNameOp(name="B", iostream=iostream2)] traces2 = [ShoutNameTrace(name="a", iostream=iostream2), ShoutNameTrace(name="b", iostream=iostream2)] # determine if running environment is multi-gpu (only needed in tf backend) strategy = tf.distribute.get_strategy() if isinstance(strategy, tf.distribute.MirroredStrategy): device_count = len(tf.config.list_physical_devices(device_type="GPU")) else: device_count = 1 for trace in traces2: trace.on_begin(None) for epoch in range(epochs): for trace in traces2: trace.on_epoch_begin(None) for batch in range(batches): for trace in traces2: trace.on_batch_begin(None) if batch == 0: # ShoutOutTrace will only be invoked the number of times equal to device count while building static # graph (for tf backend). ex: 4 GPU -> 4 times, CPU -> 1 time for _ in range(device_count): for op in ops2: op.forward(None, None) for trace in traces2: trace.on_batch_end(None) for trace in traces2: trace.on_epoch_end(None) for trace in traces2: trace.on_end(None) self.assertEqual(iostream.getvalue(), iostream2.getvalue())
class TestNetworkNetwork(unittest.TestCase): """This test has dependency: * fe.op.tensorop.model.model.ModelOp * fe.network.build """ @classmethod def setUpClass(cls): cls.tf_model = fe.build(model_fn=one_layer_tf_model, optimizer_fn=None) cls.torch_model = fe.build(model_fn=OneLayerTorchModel, optimizer_fn=None) cls.unknown_model = UnknownCompiledModel("string") def test_network_network_case_could_work(self): ops_dict = { "single tf model": [ModelOp(model=self.tf_model, inputs="x", outputs="y")], "multiple tf model": [ ModelOp(model=self.tf_model, inputs="x", outputs="y"), ModelOp(model=self.tf_model, inputs="x", outputs="y") ] } for case, ops in ops_dict.items(): with self.subTest(case): network = fe.Network(ops=ops) self.assertIsInstance(network, TFNetwork) ops_dict = { "single torch model": [ModelOp(model=self.torch_model, inputs="x", outputs="y")], "multiple torch model": [ ModelOp(model=self.torch_model, inputs="x", outputs="y"), ModelOp(model=self.torch_model, inputs="x", outputs="y") ], } for case, ops in ops_dict.items(): with self.subTest(case): network = fe.Network(ops=ops) self.assertIsInstance(network, TorchNetwork)
def get_estimator(data_dir=None, model_dir=tempfile.mkdtemp(), epochs=20, em_dim=128, batch_size=32, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None): train_ds, eval_ds, test_ds = tednmt.load_data(data_dir, translate_option="pt_to_en") pt_tokenizer = BertTokenizer.from_pretrained("neuralmind/bert-base-portuguese-cased") en_tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") pipeline = fe.Pipeline( train_data=train_ds, eval_data=eval_ds, test_data=test_ds, batch_size=batch_size, ops=[ Encode(inputs="source", outputs="source", tokenizer=pt_tokenizer), Encode(inputs="target", outputs="target", tokenizer=en_tokenizer) ], pad_value=0) model = fe.build( model_fn=lambda: Transformer(num_layers=4, em_dim=em_dim, num_heads=8, ff_dim=512, input_vocab=pt_tokenizer.vocab_size, target_vocab=en_tokenizer.vocab_size, max_pos_enc=1000, max_pos_dec=1000), optimizer_fn="adam") network = fe.Network(ops=[ ShiftData(inputs="target", outputs=("target_inp", "target_real")), CreateMasks(inputs=("source", "target_inp"), outputs=("encode_pad_mask", "decode_pad_mask", "dec_look_ahead_mask")), ModelOp(model=model, inputs=("source", "target_inp", "encode_pad_mask", "decode_pad_mask", "dec_look_ahead_mask"), outputs="pred"), MaskedCrossEntropy(inputs=("pred", "target_real"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) traces = [ MaskedAccuracy(inputs=("pred", "target_real"), outputs="masked_acc", mode="!train"), BestModelSaver(model=model, save_dir=model_dir, metric="masked_acc", save_best_mode="max"), LRScheduler(model=model, lr_fn=lambda step: lr_fn(step, em_dim)) ] estimator = fe.Estimator(pipeline=pipeline, network=network, traces=traces, epochs=epochs, max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch) return estimator
def get_estimator(): # step 1 pipeline = create_pipeline() # step 2 model = fe.build(model_fn=LeNet, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) # step 3 estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2) return estimator
def test_estimator_warmup_network_missing_key(self): loader = get_sample_tf_dataset() pipeline = fe.Pipeline(train_data=loader) # "x", "y" model = fe.build(model_fn=LeNetTf, optimizer_fn="adam") network = fe.Network(ops=[ EpochScheduler({ 1: ModelOp(model=model, inputs="x_out", outputs="y_pred"), 2: ModelOp(model=model, inputs="x_out", outputs="y_pred") }) # miss key x_out ]) est = fe.Estimator(pipeline=pipeline, network=network, epochs=2) est._prepare_traces(run_modes={"train", "eval"}) strategy = tf.distribute.get_strategy() if isinstance(strategy, tf.distribute.MirroredStrategy): with self.assertRaises(StagingError): est._warmup() else: with self.assertRaises(KeyError): est._warmup()
def test_torch_multi_output_int(self): model = fe.build(model_fn=MultiLayerTorchModel, optimizer_fn="adam") self.torch_input_data_big = self.torch_input_data_big.to( "cuda:0" if torch.cuda.is_available() else "cpu") model.to("cuda:0" if torch.cuda.is_available() else "cpu") op = ModelOp(inputs='x', outputs=['y', 'embedding'], model=model, intermediate_layers=1) op.build(framework='torch', device=torch.device( "cuda:0" if torch.cuda.is_available() else "cpu")) y, embedding = op.forward(data=self.torch_input_data_big, state=self.state) y = y.to("cpu") embedding = embedding.to('cpu') self.assertTrue( np.allclose(y.detach().numpy(), self.output_big, atol=1e-4)) self.assertTrue( np.allclose(embedding.detach().numpy(), self.embedding_output, atol=1e-4))
def get_estimator(epochs=50, batch_size=256, train_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): train_data, _ = mnist.load_data() pipeline = fe.Pipeline( train_data=train_data, batch_size=batch_size, ops=[ ExpandDims(inputs="x", outputs="x", axis=0), Normalize(inputs="x", outputs="x", mean=1.0, std=1.0, max_pixel_value=127.5), LambdaOp(fn=lambda: np.random.normal(size=[100]).astype('float32'), outputs="z") ]) gen_model = fe.build(model_fn=Generator, optimizer_fn=lambda x: torch.optim.Adam(x, lr=1e-4)) disc_model = fe.build(model_fn=Discriminator, optimizer_fn=lambda x: torch.optim.Adam(x, lr=1e-4)) network = fe.Network(ops=[ ModelOp(model=gen_model, inputs="z", outputs="x_fake"), ModelOp(model=disc_model, inputs="x_fake", outputs="fake_score"), GLoss(inputs="fake_score", outputs="gloss"), UpdateOp(model=gen_model, loss_name="gloss"), ModelOp(inputs="x", model=disc_model, outputs="true_score"), DLoss(inputs=("true_score", "fake_score"), outputs="dloss"), UpdateOp(model=disc_model, loss_name="dloss") ]) estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=ModelSaver(model=gen_model, save_dir=save_dir, frequency=5), train_steps_per_epoch=train_steps_per_epoch) return estimator
def run_test(mixed_precision, merge_grad, gradient): lr = 0.1 lr2 = 0.01 lr3 = 0.001 pipeline = fe.Pipeline(train_data=self.train_data, batch_size=4, ops=[ ExpandDims(inputs="x", outputs="x", axis=0), Minmax(inputs="x", outputs="x") ]) optimizer_fn = EpochScheduler({ 1: lambda x: torch.optim.SGD(params=x, lr=lr), 2: lambda x: torch.optim.SGD(params=x, lr=lr2), 3: lambda x: torch.optim.SGD(params=x, lr=lr3) }) model = fe.build(model_fn=LeNet_torch, optimizer_fn=optimizer_fn, mixed_precision=mixed_precision) network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), GradientOp(model=model, finals="ce", outputs="grad"), UpdateOp(model=model, loss_name="ce", gradients=gradient, merge_grad=merge_grad), ]) traces = [ CheckNetworkWeight(model=model, grad_key="grad", merge_grad=merge_grad, test_self=self, framework="torch", lrs=[lr, lr2, lr3], work_intervals=[[1, 2], [2, 3], [3, 4]]) ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=3, traces=traces, train_steps_per_epoch=2) estimator.fit(warmup=False)
def instantiate_system(): system = sample_system_object() model = fe.build(model_fn=fe.architecture.tensorflow.LeNet, optimizer_fn='adam', model_name='tf') var1 = tf.Variable(initial_value=1, trainable=True) system.network = fe.Network(ops=[ TestTensorOp( inputs="x_out", outputs="x_out", mode="train", var1=var1), ModelOp(model=model, inputs="x_out", outputs="y_pred") ]) system.traces.append(TestTrace(var1=var1)) return system
def get_estimator(level, num_augment, epochs=24, batch_size=512, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None): assert 0 <= level <= 10, "the level should be between 0 and 10" train_data, test_data = load_data() aug_ops = [ OneOf( Rotate(level=level, inputs="x", outputs="x", mode="train"), Identity(level=level, inputs="x", outputs="x", mode="train"), AutoContrast(level=level, inputs="x", outputs="x", mode="train"), Equalize(level=level, inputs="x", outputs="x", mode="train"), Posterize(level=level, inputs="x", outputs="x", mode="train"), Solarize(level=level, inputs="x", outputs="x", mode="train"), Sharpness(level=level, inputs="x", outputs="x", mode="train"), Contrast(level=level, inputs="x", outputs="x", mode="train"), Color(level=level, inputs="x", outputs="x", mode="train"), Brightness(level=level, inputs="x", outputs="x", mode="train"), ShearX(level=level, inputs="x", outputs="x", mode="train"), ShearY(level=level, inputs="x", outputs="x", mode="train"), TranslateX(level=level, inputs="x", outputs="x", mode="train"), TranslateY(level=level, inputs="x", outputs="x", mode="train"), ) for _ in range(num_augment) ] pipeline = fe.Pipeline(train_data=train_data, test_data=test_data, batch_size=batch_size, ops=aug_ops + [ Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616)), ChannelTranspose(inputs="x", outputs="x"), ]) model = fe.build(model_fn=MyModel, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) estimator = fe.Estimator( pipeline=pipeline, network=network, epochs=epochs, traces=Accuracy(true_key="y", pred_key="y_pred"), max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch) return estimator
def instantiate_system(): system = sample_system_object_torch() model = fe.build(model_fn=fe.architecture.pytorch.LeNet, optimizer_fn='adam', model_name='torch') var1 = torch.tensor(1.0) system.network = fe.Network(ops=[ TestTensorOp( inputs="x_out", outputs="x_out", mode="train", var1=var1), ModelOp(model=model, inputs="x_out", outputs="y_pred") ]) system.traces.append(TestTrace(var1=var1)) return system
def sample_system_object_torch(): x_train = np.random.rand(3, 28, 28, 3) y_train = np.random.randint(10, size=(3, )) x_eval = np.random.rand(2, 28, 28, 3) y_eval = np.random.randint(10, size=(2, )) train_data = NumpyDataset({'x': x_train, 'y': y_train}) eval_data = NumpyDataset({'x': x_eval, 'y': y_eval}) test_data = eval_data.split(0.5) model = fe.build(model_fn=fe.architecture.pytorch.LeNet, optimizer_fn='adam', model_name='torch') pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=1) network = fe.Network(ops=[ModelOp(model=model, inputs="x_out", outputs="y_pred")]) system = System(network=network, pipeline=pipeline, traces=[], total_epochs=10, mode='train') return system
def get_estimator(max_len=20, epochs=10, batch_size=64, max_train_steps_per_epoch=None, max_eval_steps_per_epoch=None, pretrained_model='bert-base-uncased', save_dir=tempfile.mkdtemp(), data_dir=None): # step 1 prepare data train_data, eval_data, data_vocab, label_vocab = german_ner.load_data(root_dir=data_dir) tokenizer = BertTokenizer.from_pretrained(pretrained_model, do_lower_case=True) tag2idx = char2idx(label_vocab) pipeline = fe.Pipeline( train_data=train_data, eval_data=eval_data, batch_size=batch_size, ops=[ Tokenize(inputs="x", outputs="x", tokenize_fn=tokenizer.tokenize), WordtoId(inputs="x", outputs="x", mapping=tokenizer.convert_tokens_to_ids), WordtoId(inputs="y", outputs="y", mapping=tag2idx), PadSequence(max_len=max_len, inputs="x", outputs="x"), PadSequence(max_len=max_len, value=len(tag2idx), inputs="y", outputs="y"), AttentionMask(inputs="x", outputs="x_masks") ]) # step 2. prepare model bert_config = BertConfig.from_pretrained(pretrained_model) num_hidden_layers = bert_config.to_dict()['num_hidden_layers'] head_masks = [None] * num_hidden_layers model = fe.build(model_fn=lambda: NERModel(head_masks=head_masks, pretrained_model=pretrained_model), optimizer_fn=lambda x: torch.optim.Adam(x, lr=1e-5)) network = fe.Network(ops=[ ModelOp(model=model, inputs=["x", "x_masks"], outputs="y_pred"), Reshape(inputs="y", outputs="y", shape=(-1, )), Reshape(inputs="y_pred", outputs="y_pred", shape=(-1, 24)), CrossEntropy(inputs=("y_pred", "y"), outputs="loss"), UpdateOp(model=model, loss_name="loss") ]) traces = [Accuracy(true_key="y", pred_key="y_pred"), BestModelSaver(model=model, save_dir=save_dir)] # step 3 prepare estimator estimator = fe.Estimator(network=network, pipeline=pipeline, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch, max_eval_steps_per_epoch=max_eval_steps_per_epoch) return estimator
def test_estimator_warmup_network_missing_key(self): loader = get_sample_tf_dataset() pipeline = fe.Pipeline(train_data=loader) # "x", "y" model = fe.build(model_fn=LeNetTf, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x_out", outputs="y_pred") # miss key x_out ]) est = fe.Estimator(pipeline=pipeline, network=network, epochs=1) est._prepare_traces(run_modes={"train", "eval"}) with self.assertRaises(KeyError): est._warmup(warmup=True)
def instantiate_system(): system = sample_system_object() model = fe.build(model_fn=fe.architecture.tensorflow.LeNet, optimizer_fn='adam', model_name='tf') system.network = fe.Network(ops=[ fe.op.tensorop.meta.OneOf( TestTensorOp( inputs="x_out", outputs="x_out", mode="train", var=1), TestTensorOp( inputs="x_out", outputs="x_out", mode="train", var=1)), ModelOp(model=model, inputs="x_out", outputs="y_pred") ]) return system
def test_saliency(self): fe.estimator.enable_deterministic(200) label_mapping = { 'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4, 'dog': 5, 'frog': 6, 'horse': 7, 'ship': 8, 'truck': 9 } batch_size = 32 train_data, eval_data = cifar10.load_data() pipeline = fe.Pipeline(test_data=train_data, batch_size=batch_size, ops=[Normalize(inputs="x", outputs="x")], num_process=0) weight_path = os.path.abspath(os.path.join(__file__, "..", "resources", "lenet_cifar10_tf.h5")) model = fe.build(model_fn=lambda: LeNet(input_shape=(32, 32, 3)), optimizer_fn="adam", weights_path=weight_path) network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred") ]) save_dir = tempfile.mkdtemp() traces = [ Saliency(model=model, model_inputs="x", class_key="y", model_outputs="y_pred", samples=5, label_mapping=label_mapping), ImageSaver(inputs="saliency", save_dir=save_dir) ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=5, traces=traces, log_steps=1000) estimator.test() ans_img_path = os.path.abspath(os.path.join(__file__, "..", "resources", "saliency_figure.png")) ans_img = img_to_rgb_array(ans_img_path) output_img_path = os.path.join(save_dir, "saliency_test_epoch_5.png") output_img = img_to_rgb_array(output_img_path) self.assertTrue(check_img_similar(output_img, ans_img))
def get_estimator(batch_size=100, epochs=20, max_train_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): train_data, _ = load_data() pipeline = fe.Pipeline( train_data=train_data, batch_size=batch_size, ops=[ ExpandDims(inputs="x", outputs="x", axis=0), Minmax(inputs="x", outputs="x"), Binarize(inputs="x", outputs="x", threshold=0.5), ]) encode_model = fe.build(model_fn=EncoderNet, optimizer_fn="adam", model_name="encoder") decode_model = fe.build(model_fn=DecoderNet, optimizer_fn="adam", model_name="decoder") network = fe.Network(ops=[ ModelOp(model=encode_model, inputs="x", outputs="meanlogvar"), SplitOp(inputs="meanlogvar", outputs=("mean", "logvar")), ReparameterizepOp(inputs=("mean", "logvar"), outputs="z"), ModelOp(model=decode_model, inputs="z", outputs="x_logit"), CrossEntropy(inputs=("x_logit", "x"), outputs="cross_entropy"), CVAELoss(inputs=("cross_entropy", "mean", "logvar", "z"), outputs="loss"), UpdateOp(model=encode_model, loss_name="loss"), UpdateOp(model=decode_model, loss_name="loss"), ]) traces = [ BestModelSaver(model=encode_model, save_dir=save_dir), BestModelSaver(model=decode_model, save_dir=save_dir) ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, max_train_steps_per_epoch=max_train_steps_per_epoch) return estimator
def test_tf_model_end_to_end_gradient(self): train_data, _ = mnist.load_data() pipeline = fe.Pipeline(train_data=train_data, batch_size=4, ops=[ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x")]) model = fe.build(model_fn=LeNet_tf, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), GradientOp(model=model, finals="ce", outputs="gradients"), UpdateOp(model=model, gradients="gradients", loss_name="ce") ]) estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2, max_train_steps_per_epoch=10) estimator.fit()
def get_estimator(epochs=30, batch_size=128, seq_length=20, vocab_size=10000, data_dir=None, train_steps_per_epoch=None, save_dir=tempfile.mkdtemp()): train_data, eval_data, _, _ = load_data(root_dir=data_dir, seq_length=seq_length + 1) pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, ops=[ CreateInputAndTarget(inputs="x", outputs=("x", "y")), Batch(batch_size=batch_size, drop_last=True) ]) # step 2 model = fe.build( model_fn=lambda: BuildModel( vocab_size, embedding_dim=300, rnn_units=600), optimizer_fn=lambda x: torch.optim.SGD(x, lr=1.0, momentum=0.9)) network = fe.Network(ops=[ DimesionAdjust(inputs=("x", "y"), outputs=("x", "y")), ModelOp(model=model, inputs="x", outputs="y_pred", mode=None), CrossEntropy(inputs=("y_pred", "y"), outputs="ce", form="sparse", from_logits=True), UpdateOp(model=model, loss_name="ce") ]) # step 3 traces = [ Perplexity(inputs="ce", outputs="perplexity", mode="eval"), LRScheduler(model=model, lr_fn=lambda step: lr_schedule(step, init_lr=1.0)), BestModelSaver(model=model, save_dir=save_dir, metric='perplexity', save_best_mode='min', load_best_final=True), EarlyStopping(monitor="perplexity", patience=5) ] estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces, train_steps_per_epoch=train_steps_per_epoch) return estimator
def get_estimator(): ds, _ = mnist.load_data() ds = NegativeImageSimulatedTube(ds) pipeline = fe.Pipeline(train_data=ds, ops=[ ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x") ]) model = fe.build(model_fn=LeNet, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2) return estimator
def get_estimator(): pos_real, _ = mnist.load_data() neg_real, _ = mnist.load_data() neg_sim, _ = mnist.load_data() neg_sim = NegativeImageSimulatedTube(neg_sim) batch_ds = BatchDataset(datasets=(pos_real, neg_real, neg_sim), num_samples=(2, 2, 1)) pipeline = fe.Pipeline(train_data=batch_ds, ops=[ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x")]) model = fe.build(model_fn=LeNet, optimizer_fn="adam") network = fe.Network(ops=[ ModelOp(model=model, inputs="x", outputs="y_pred"), CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), UpdateOp(model=model, loss_name="ce") ]) estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2) return estimator
def test_network_collect_model_with_model_op_and_update_op(self): model_fns = {"tf_model_fn": one_layer_tf_model, "torch_model_fn": OneLayerTorchModel} for name, model_fn in model_fns.items(): with self.subTest(name): model = fe.build(model_fn=model_fn, optimizer_fn=None) model2 = fe.build(model_fn=model_fn, optimizer_fn=None) ops = [ SampleTensorOp(inputs="x", outputs="x"), ModelOp(model=model, inputs="x", outputs="y2"), UpdateOp(model=model2, loss_name="ce") ] models = fe.network._collect_models(ops) ans = {model, model2} self.assertEqual(models, ans)