def get_estimator(epsilon=0.04,
epochs=10,
batch_size=32,
max_train_steps_per_epoch=None,
max_eval_steps_per_epoch=None,
save_dir=tempfile.mkdtemp()):
# step 1
train_data, eval_data = cifair10.load_data()
test_data = eval_data.split(0.5)
pipeline = fe.Pipeline(
train_data=train_data,
eval_data=eval_data,
test_data=test_data,
batch_size=batch_size,
ops=[Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616))])
# step 2
model = fe.build(model_fn=lambda: LeNet(input_shape=(32, 32, 3)), optimizer_fn="adam")
network = fe.Network(ops=[
Watch(inputs="x"),
ModelOp(model=model, inputs="x", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="base_ce"),
FGSM(data="x", loss="base_ce", outputs="x_adverse", epsilon=epsilon),
ModelOp(model=model, inputs="x_adverse", outputs="y_pred_adv"),
CrossEntropy(inputs=("y_pred_adv", "y"), outputs="adv_ce"),
Average(inputs=("base_ce", "adv_ce"), outputs="avg_ce"),
UpdateOp(model=model, loss_name="avg_ce")
])
# step 3
traces = [
Accuracy(true_key="y", pred_key="y_pred", output_name="base accuracy"),
Accuracy(true_key="y", pred_key="y_pred_adv", output_name="adversarial accuracy"),
BestModelSaver(model=model, save_dir=save_dir, metric="adv_ce", save_best_mode="min"),
]
estimator = fe.Estimator(pipeline=pipeline,
network=network,
epochs=epochs,
traces=traces,
max_train_steps_per_epoch=max_train_steps_per_epoch,
max_eval_steps_per_epoch=max_eval_steps_per_epoch,
monitor_names=["base_ce", "adv_ce"])
return estimator
def get_estimator(batch_size=4,
epochs=2,
max_train_steps_per_epoch=None,
log_steps=100,
style_weight=5.0,
content_weight=1.0,
tv_weight=1e-4,
save_dir=tempfile.mkdtemp(),
style_img_path='Vassily_Kandinsky,_1913_-_Composition_7.jpg',
data_dir=None):
train_data, _ = mscoco.load_data(root_dir=data_dir, load_bboxes=False, load_masks=False, load_captions=False)
device = "cuda" if torch.cuda.is_available() else "cpu"
style_img = cv2.imread(style_img_path)
assert style_img is not None, "cannot load the style image, please go to the folder with style image"
style_img = cv2.resize(style_img, (256, 256))
style_img = (style_img.astype(np.float32) - 127.5) / 127.5
pipeline = fe.Pipeline(
train_data=train_data,
batch_size=batch_size,
ops=[
ReadImage(inputs="image", outputs="image"),
Normalize(inputs="image", outputs="image", mean=1.0, std=1.0, max_pixel_value=127.5),
Resize(height=256, width=256, image_in="image", image_out="image"),
LambdaOp(fn=lambda: style_img, outputs="style_image"),
ChannelTranspose(inputs=["image", "style_image"], outputs=["image", "style_image"])
])
model = fe.build(model_fn=StyleTransferNet,
model_name="style_transfer_net",
optimizer_fn=lambda x: torch.optim.Adam(x, lr=1e-3))
network = fe.Network(ops=[
ModelOp(inputs="image", model=model, outputs="image_out"),
ExtractVGGFeatures(inputs="style_image", outputs="y_style", device=device),
ExtractVGGFeatures(inputs="image", outputs="y_content", device=device),
ExtractVGGFeatures(inputs="image_out", outputs="y_pred", device=device),
StyleContentLoss(style_weight=style_weight,
content_weight=content_weight,
tv_weight=tv_weight,
inputs=('y_pred', 'y_style', 'y_content', 'image_out'),
outputs='loss'),
UpdateOp(model=model, loss_name="loss")
])
estimator = fe.Estimator(network=network,
pipeline=pipeline,
traces=ModelSaver(model=model, save_dir=save_dir, frequency=1),
epochs=epochs,
max_train_steps_per_epoch=max_train_steps_per_epoch,
log_steps=log_steps)
return estimator
def get_estimator(epochs=2,
batch_size=32,
max_train_steps_per_epoch=None,
max_eval_steps_per_epoch=None,
save_dir=tempfile.mkdtemp()):
# step 1
train_data, eval_data = mnist.load_data()
test_data = eval_data.split(0.5)
pipeline = fe.Pipeline(train_data=train_data,
eval_data=eval_data,
test_data=test_data,
batch_size=batch_size,
ops=[
ExpandDims(inputs="x", outputs="x"),
Minmax(inputs="x", outputs="x")
],
num_process=0)
# step 2
model = fe.build(model_fn=LeNet, optimizer_fn="adam")
print([f"{idx}: {x.name}" for idx, x in enumerate(model.submodules)])
network = fe.Network(ops=[
Watch(inputs="x"),
ModelOp(model=model,
inputs="x",
outputs=["y_pred", "embedding"],
intermediate_layers='dense'),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce"),
GradientOp(finals="embedding", inputs="x", outputs="grads"),
UpdateOp(model=model, loss_name="ce")
])
# step 3
traces = [
Accuracy(true_key="y", pred_key="y_pred"),
Inspector(),
BestModelSaver(model=model,
save_dir=save_dir,
metric="accuracy",
save_best_mode="max"),
LRScheduler(model=model,
lr_fn=lambda step: cosine_decay(
step, cycle_length=3750, init_lr=1e-3)),
TensorBoard(log_dir="tf_logs",
write_embeddings="embedding",
embedding_labels="y")
]
estimator = fe.Estimator(
pipeline=pipeline,
network=network,
epochs=epochs,
traces=traces,
max_train_steps_per_epoch=max_train_steps_per_epoch,
max_eval_steps_per_epoch=max_eval_steps_per_epoch)
return estimator
def test_estimator_warmup_trace_missing_key(self):
loader = get_sample_tf_dataset()
pipeline = fe.Pipeline(train_data=loader) # "x", "y"
model = fe.build(model_fn=LeNetTf, optimizer_fn="adam")
network = fe.Network(ops=[
EpochScheduler(
{
1: ModelOp(model=model, inputs="x", outputs="y_pred"),
2: ModelOp(model=model, inputs="x", outputs="y_pred")
}) # miss key x_out
])
est = fe.Estimator(pipeline=pipeline,
network=network,
epochs=2,
traces=[Trace(inputs="z")]) # miss key "z"
est._prepare_traces(run_modes={"train", "eval"})
with self.assertRaises(AssertionError):
est._warmup()
def test_estimator_warmup_torch_dataset_tf_model_smoke(self):
loader = get_sample_torch_dataloader(expand_axis=-1)
pipeline = fe.Pipeline(train_data=loader) # "x", "y"
model = fe.build(model_fn=LeNetTf, optimizer_fn="adam")
network = fe.Network(ops=[ModelOp(model=model, inputs="x", outputs="y_pred")])
est = fe.Estimator(pipeline=pipeline, network=network, epochs=1, traces=[Trace(inputs="y_pred")])
est._prepare_traces(run_modes={"train", "eval"})
est._warmup()
self.assertTrue(True)
def setUpClass(cls):
train_data, eval_data = mnist.load_data()
cls.pipeline = fe.Pipeline(train_data=train_data)
model = fe.build(model_fn=LeNetTf, optimizer_fn="adam")
cls.network = fe.Network(ops=[
ModelOp(model=model, inputs="x_out", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce"),
UpdateOp(model=model, loss_name="ce")
])
def get_estimator(epochs=50,
batch_size=128,
max_train_steps_per_epoch=None,
max_eval_steps_per_epoch=None,
save_dir=tempfile.mkdtemp()):
# step 1
train_data, eval_data = cifair100.load_data()
# Add label noise to simulate real-world labeling problems
corrupt_dataset(train_data)
test_data = eval_data.split(range(len(eval_data) // 2))
pipeline = fe.Pipeline(
train_data=train_data,
eval_data=eval_data,
test_data=test_data,
batch_size=batch_size,
ops=[
Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616)),
PadIfNeeded(min_height=40, min_width=40, image_in="x", image_out="x", mode="train"),
RandomCrop(32, 32, image_in="x", image_out="x", mode="train"),
Sometimes(HorizontalFlip(image_in="x", image_out="x", mode="train")),
CoarseDropout(inputs="x", outputs="x", mode="train", max_holes=1),
ChannelTranspose(inputs="x", outputs="x")
])
# step 2
model = fe.build(model_fn=big_lenet, optimizer_fn='adam')
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
SuperLoss(CrossEntropy(inputs=("y_pred", "y"), outputs="ce"), output_confidence="confidence"),
UpdateOp(model=model, loss_name="ce")
])
# step 3
traces = [
MCC(true_key="y", pred_key="y_pred"),
BestModelSaver(model=model, save_dir=save_dir, metric="mcc", save_best_mode="max", load_best_final=True),
LabelTracker(metric="confidence",
label="data_labels",
label_mapping={
"Normal": 0, "Corrupted": 1
},
mode="train",
outputs="label_confidence"),
ImageSaver(inputs="label_confidence", save_dir=save_dir, mode="train"),
]
estimator = fe.Estimator(pipeline=pipeline,
network=network,
epochs=epochs,
traces=traces,
max_train_steps_per_epoch=max_train_steps_per_epoch,
max_eval_steps_per_epoch=max_eval_steps_per_epoch)
return estimator
def instantiate_system():
system = sample_system_object_torch()
model = fe.build(model_fn=fe.architecture.pytorch.LeNet,
optimizer_fn='adam',
model_name='torch')
system.network = fe.Network(ops=[
fe.op.tensorop.meta.Repeat(op=TestTensorOp(
inputs="x_out", outputs="x_out", mode="train", var=1),
repeat=2),
ModelOp(model=model, inputs="x_out", outputs="y_pred")
])
return system
def test_estimator_check_network_op_trace_invoke_sequence_tf_backend(self):
epochs = 1
batches = 10 # dataset has 100 sample, and batch_size is 10
iostream = StringIO()
loader = get_sample_tf_dataset()
pipeline = fe.Pipeline(test_data=loader)
model = fe.build(model_fn=LeNetTf, optimizer_fn="adam")
ops = [
ModelOp(model=model, inputs="x", outputs="y_pred"),
ShoutNameOp(name="A", iostream=iostream),
ShoutNameOp(name="B", iostream=iostream)
]
network = fe.Network(ops=ops)
traces = [ShoutNameTrace(name="a", iostream=iostream), ShoutNameTrace(name="b", iostream=iostream)]
est = fe.Estimator(pipeline=pipeline, network=network, epochs=epochs, traces=traces)
est.test()
# create the expected calling sequence in another iostream
iostream2 = StringIO()
ops2 = [ShoutNameOp(name="A", iostream=iostream2), ShoutNameOp(name="B", iostream=iostream2)]
traces2 = [ShoutNameTrace(name="a", iostream=iostream2), ShoutNameTrace(name="b", iostream=iostream2)]
# determine if running environment is multi-gpu (only needed in tf backend)
strategy = tf.distribute.get_strategy()
if isinstance(strategy, tf.distribute.MirroredStrategy):
device_count = len(tf.config.list_physical_devices(device_type="GPU"))
else:
device_count = 1
for trace in traces2:
trace.on_begin(None)
for epoch in range(epochs):
for trace in traces2:
trace.on_epoch_begin(None)
for batch in range(batches):
for trace in traces2:
trace.on_batch_begin(None)
if batch == 0:
# ShoutOutTrace will only be invoked the number of times equal to device count while building static
# graph (for tf backend). ex: 4 GPU -> 4 times, CPU -> 1 time
for _ in range(device_count):
for op in ops2:
op.forward(None, None)
for trace in traces2:
trace.on_batch_end(None)
for trace in traces2:
trace.on_epoch_end(None)
for trace in traces2:
trace.on_end(None)
self.assertEqual(iostream.getvalue(), iostream2.getvalue())
class TestNetworkNetwork(unittest.TestCase):
"""This test has dependency:
* fe.op.tensorop.model.model.ModelOp
* fe.network.build
"""
@classmethod
def setUpClass(cls):
cls.tf_model = fe.build(model_fn=one_layer_tf_model, optimizer_fn=None)
cls.torch_model = fe.build(model_fn=OneLayerTorchModel,
optimizer_fn=None)
cls.unknown_model = UnknownCompiledModel("string")
def test_network_network_case_could_work(self):
ops_dict = {
"single tf model":
[ModelOp(model=self.tf_model, inputs="x", outputs="y")],
"multiple tf model": [
ModelOp(model=self.tf_model, inputs="x", outputs="y"),
ModelOp(model=self.tf_model, inputs="x", outputs="y")
]
}
for case, ops in ops_dict.items():
with self.subTest(case):
network = fe.Network(ops=ops)
self.assertIsInstance(network, TFNetwork)
ops_dict = {
"single torch model":
[ModelOp(model=self.torch_model, inputs="x", outputs="y")],
"multiple torch model": [
ModelOp(model=self.torch_model, inputs="x", outputs="y"),
ModelOp(model=self.torch_model, inputs="x", outputs="y")
],
}
for case, ops in ops_dict.items():
with self.subTest(case):
network = fe.Network(ops=ops)
self.assertIsInstance(network, TorchNetwork)
def get_estimator(data_dir=None,
model_dir=tempfile.mkdtemp(),
epochs=20,
em_dim=128,
batch_size=32,
max_train_steps_per_epoch=None,
max_eval_steps_per_epoch=None):
train_ds, eval_ds, test_ds = tednmt.load_data(data_dir, translate_option="pt_to_en")
pt_tokenizer = BertTokenizer.from_pretrained("neuralmind/bert-base-portuguese-cased")
en_tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
pipeline = fe.Pipeline(
train_data=train_ds,
eval_data=eval_ds,
test_data=test_ds,
batch_size=batch_size,
ops=[
Encode(inputs="source", outputs="source", tokenizer=pt_tokenizer),
Encode(inputs="target", outputs="target", tokenizer=en_tokenizer)
],
pad_value=0)
model = fe.build(
model_fn=lambda: Transformer(num_layers=4,
em_dim=em_dim,
num_heads=8,
ff_dim=512,
input_vocab=pt_tokenizer.vocab_size,
target_vocab=en_tokenizer.vocab_size,
max_pos_enc=1000,
max_pos_dec=1000),
optimizer_fn="adam")
network = fe.Network(ops=[
ShiftData(inputs="target", outputs=("target_inp", "target_real")),
CreateMasks(inputs=("source", "target_inp"),
outputs=("encode_pad_mask", "decode_pad_mask", "dec_look_ahead_mask")),
ModelOp(model=model,
inputs=("source", "target_inp", "encode_pad_mask", "decode_pad_mask", "dec_look_ahead_mask"),
outputs="pred"),
MaskedCrossEntropy(inputs=("pred", "target_real"), outputs="ce"),
UpdateOp(model=model, loss_name="ce")
])
traces = [
MaskedAccuracy(inputs=("pred", "target_real"), outputs="masked_acc", mode="!train"),
BestModelSaver(model=model, save_dir=model_dir, metric="masked_acc", save_best_mode="max"),
LRScheduler(model=model, lr_fn=lambda step: lr_fn(step, em_dim))
]
estimator = fe.Estimator(pipeline=pipeline,
network=network,
traces=traces,
epochs=epochs,
max_train_steps_per_epoch=max_train_steps_per_epoch,
max_eval_steps_per_epoch=max_eval_steps_per_epoch)
return estimator
def get_estimator():
# step 1
pipeline = create_pipeline()
# step 2
model = fe.build(model_fn=LeNet, optimizer_fn="adam")
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce"),
UpdateOp(model=model, loss_name="ce")
])
# step 3
estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2)
return estimator
def test_estimator_warmup_network_missing_key(self):
loader = get_sample_tf_dataset()
pipeline = fe.Pipeline(train_data=loader) # "x", "y"
model = fe.build(model_fn=LeNetTf, optimizer_fn="adam")
network = fe.Network(ops=[
EpochScheduler({
1: ModelOp(model=model, inputs="x_out", outputs="y_pred"),
2: ModelOp(model=model, inputs="x_out", outputs="y_pred")
}) # miss key x_out
])
est = fe.Estimator(pipeline=pipeline, network=network, epochs=2)
est._prepare_traces(run_modes={"train", "eval"})
strategy = tf.distribute.get_strategy()
if isinstance(strategy, tf.distribute.MirroredStrategy):
with self.assertRaises(StagingError):
est._warmup()
else:
with self.assertRaises(KeyError):
est._warmup()
def test_torch_multi_output_int(self):
model = fe.build(model_fn=MultiLayerTorchModel, optimizer_fn="adam")
self.torch_input_data_big = self.torch_input_data_big.to(
"cuda:0" if torch.cuda.is_available() else "cpu")
model.to("cuda:0" if torch.cuda.is_available() else "cpu")
op = ModelOp(inputs='x',
outputs=['y', 'embedding'],
model=model,
intermediate_layers=1)
op.build(framework='torch',
device=torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu"))
y, embedding = op.forward(data=self.torch_input_data_big,
state=self.state)
y = y.to("cpu")
embedding = embedding.to('cpu')
self.assertTrue(
np.allclose(y.detach().numpy(), self.output_big, atol=1e-4))
self.assertTrue(
np.allclose(embedding.detach().numpy(),
self.embedding_output,
atol=1e-4))
def get_estimator(epochs=50,
batch_size=256,
train_steps_per_epoch=None,
save_dir=tempfile.mkdtemp()):
train_data, _ = mnist.load_data()
pipeline = fe.Pipeline(
train_data=train_data,
batch_size=batch_size,
ops=[
ExpandDims(inputs="x", outputs="x", axis=0),
Normalize(inputs="x",
outputs="x",
mean=1.0,
std=1.0,
max_pixel_value=127.5),
LambdaOp(fn=lambda: np.random.normal(size=[100]).astype('float32'),
outputs="z")
])
gen_model = fe.build(model_fn=Generator,
optimizer_fn=lambda x: torch.optim.Adam(x, lr=1e-4))
disc_model = fe.build(model_fn=Discriminator,
optimizer_fn=lambda x: torch.optim.Adam(x, lr=1e-4))
network = fe.Network(ops=[
ModelOp(model=gen_model, inputs="z", outputs="x_fake"),
ModelOp(model=disc_model, inputs="x_fake", outputs="fake_score"),
GLoss(inputs="fake_score", outputs="gloss"),
UpdateOp(model=gen_model, loss_name="gloss"),
ModelOp(inputs="x", model=disc_model, outputs="true_score"),
DLoss(inputs=("true_score", "fake_score"), outputs="dloss"),
UpdateOp(model=disc_model, loss_name="dloss")
])
estimator = fe.Estimator(pipeline=pipeline,
network=network,
epochs=epochs,
traces=ModelSaver(model=gen_model,
save_dir=save_dir,
frequency=5),
train_steps_per_epoch=train_steps_per_epoch)
return estimator
def run_test(mixed_precision, merge_grad, gradient):
lr = 0.1
lr2 = 0.01
lr3 = 0.001
pipeline = fe.Pipeline(train_data=self.train_data,
batch_size=4,
ops=[
ExpandDims(inputs="x",
outputs="x",
axis=0),
Minmax(inputs="x", outputs="x")
])
optimizer_fn = EpochScheduler({
1:
lambda x: torch.optim.SGD(params=x, lr=lr),
2:
lambda x: torch.optim.SGD(params=x, lr=lr2),
3:
lambda x: torch.optim.SGD(params=x, lr=lr3)
})
model = fe.build(model_fn=LeNet_torch,
optimizer_fn=optimizer_fn,
mixed_precision=mixed_precision)
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce"),
GradientOp(model=model, finals="ce", outputs="grad"),
UpdateOp(model=model,
loss_name="ce",
gradients=gradient,
merge_grad=merge_grad),
])
traces = [
CheckNetworkWeight(model=model,
grad_key="grad",
merge_grad=merge_grad,
test_self=self,
framework="torch",
lrs=[lr, lr2, lr3],
work_intervals=[[1, 2], [2, 3], [3, 4]])
]
estimator = fe.Estimator(pipeline=pipeline,
network=network,
epochs=3,
traces=traces,
train_steps_per_epoch=2)
estimator.fit(warmup=False)
def instantiate_system():
system = sample_system_object()
model = fe.build(model_fn=fe.architecture.tensorflow.LeNet,
optimizer_fn='adam',
model_name='tf')
var1 = tf.Variable(initial_value=1, trainable=True)
system.network = fe.Network(ops=[
TestTensorOp(
inputs="x_out", outputs="x_out", mode="train", var1=var1),
ModelOp(model=model, inputs="x_out", outputs="y_pred")
])
system.traces.append(TestTrace(var1=var1))
return system
def get_estimator(level,
num_augment,
epochs=24,
batch_size=512,
max_train_steps_per_epoch=None,
max_eval_steps_per_epoch=None):
assert 0 <= level <= 10, "the level should be between 0 and 10"
train_data, test_data = load_data()
aug_ops = [
OneOf(
Rotate(level=level, inputs="x", outputs="x", mode="train"),
Identity(level=level, inputs="x", outputs="x", mode="train"),
AutoContrast(level=level, inputs="x", outputs="x", mode="train"),
Equalize(level=level, inputs="x", outputs="x", mode="train"),
Posterize(level=level, inputs="x", outputs="x", mode="train"),
Solarize(level=level, inputs="x", outputs="x", mode="train"),
Sharpness(level=level, inputs="x", outputs="x", mode="train"),
Contrast(level=level, inputs="x", outputs="x", mode="train"),
Color(level=level, inputs="x", outputs="x", mode="train"),
Brightness(level=level, inputs="x", outputs="x", mode="train"),
ShearX(level=level, inputs="x", outputs="x", mode="train"),
ShearY(level=level, inputs="x", outputs="x", mode="train"),
TranslateX(level=level, inputs="x", outputs="x", mode="train"),
TranslateY(level=level, inputs="x", outputs="x", mode="train"),
) for _ in range(num_augment)
]
pipeline = fe.Pipeline(train_data=train_data,
test_data=test_data,
batch_size=batch_size,
ops=aug_ops + [
Normalize(inputs="x",
outputs="x",
mean=(0.4914, 0.4822, 0.4465),
std=(0.2471, 0.2435, 0.2616)),
ChannelTranspose(inputs="x", outputs="x"),
])
model = fe.build(model_fn=MyModel, optimizer_fn="adam")
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce"),
UpdateOp(model=model, loss_name="ce")
])
estimator = fe.Estimator(
pipeline=pipeline,
network=network,
epochs=epochs,
traces=Accuracy(true_key="y", pred_key="y_pred"),
max_train_steps_per_epoch=max_train_steps_per_epoch,
max_eval_steps_per_epoch=max_eval_steps_per_epoch)
return estimator
def instantiate_system():
system = sample_system_object_torch()
model = fe.build(model_fn=fe.architecture.pytorch.LeNet,
optimizer_fn='adam',
model_name='torch')
var1 = torch.tensor(1.0)
system.network = fe.Network(ops=[
TestTensorOp(
inputs="x_out", outputs="x_out", mode="train", var1=var1),
ModelOp(model=model, inputs="x_out", outputs="y_pred")
])
system.traces.append(TestTrace(var1=var1))
return system
def sample_system_object_torch():
x_train = np.random.rand(3, 28, 28, 3)
y_train = np.random.randint(10, size=(3, ))
x_eval = np.random.rand(2, 28, 28, 3)
y_eval = np.random.randint(10, size=(2, ))
train_data = NumpyDataset({'x': x_train, 'y': y_train})
eval_data = NumpyDataset({'x': x_eval, 'y': y_eval})
test_data = eval_data.split(0.5)
model = fe.build(model_fn=fe.architecture.pytorch.LeNet, optimizer_fn='adam', model_name='torch')
pipeline = fe.Pipeline(train_data=train_data, eval_data=eval_data, test_data=test_data, batch_size=1)
network = fe.Network(ops=[ModelOp(model=model, inputs="x_out", outputs="y_pred")])
system = System(network=network, pipeline=pipeline, traces=[], total_epochs=10, mode='train')
return system
def get_estimator(max_len=20,
epochs=10,
batch_size=64,
max_train_steps_per_epoch=None,
max_eval_steps_per_epoch=None,
pretrained_model='bert-base-uncased',
save_dir=tempfile.mkdtemp(),
data_dir=None):
# step 1 prepare data
train_data, eval_data, data_vocab, label_vocab = german_ner.load_data(root_dir=data_dir)
tokenizer = BertTokenizer.from_pretrained(pretrained_model, do_lower_case=True)
tag2idx = char2idx(label_vocab)
pipeline = fe.Pipeline(
train_data=train_data,
eval_data=eval_data,
batch_size=batch_size,
ops=[
Tokenize(inputs="x", outputs="x", tokenize_fn=tokenizer.tokenize),
WordtoId(inputs="x", outputs="x", mapping=tokenizer.convert_tokens_to_ids),
WordtoId(inputs="y", outputs="y", mapping=tag2idx),
PadSequence(max_len=max_len, inputs="x", outputs="x"),
PadSequence(max_len=max_len, value=len(tag2idx), inputs="y", outputs="y"),
AttentionMask(inputs="x", outputs="x_masks")
])
# step 2. prepare model
bert_config = BertConfig.from_pretrained(pretrained_model)
num_hidden_layers = bert_config.to_dict()['num_hidden_layers']
head_masks = [None] * num_hidden_layers
model = fe.build(model_fn=lambda: NERModel(head_masks=head_masks, pretrained_model=pretrained_model),
optimizer_fn=lambda x: torch.optim.Adam(x, lr=1e-5))
network = fe.Network(ops=[
ModelOp(model=model, inputs=["x", "x_masks"], outputs="y_pred"),
Reshape(inputs="y", outputs="y", shape=(-1, )),
Reshape(inputs="y_pred", outputs="y_pred", shape=(-1, 24)),
CrossEntropy(inputs=("y_pred", "y"), outputs="loss"),
UpdateOp(model=model, loss_name="loss")
])
traces = [Accuracy(true_key="y", pred_key="y_pred"), BestModelSaver(model=model, save_dir=save_dir)]
# step 3 prepare estimator
estimator = fe.Estimator(network=network,
pipeline=pipeline,
epochs=epochs,
traces=traces,
max_train_steps_per_epoch=max_train_steps_per_epoch,
max_eval_steps_per_epoch=max_eval_steps_per_epoch)
return estimator
def test_estimator_warmup_network_missing_key(self):
loader = get_sample_tf_dataset()
pipeline = fe.Pipeline(train_data=loader) # "x", "y"
model = fe.build(model_fn=LeNetTf, optimizer_fn="adam")
network = fe.Network(ops=[
ModelOp(model=model, inputs="x_out",
outputs="y_pred") # miss key x_out
])
est = fe.Estimator(pipeline=pipeline, network=network, epochs=1)
est._prepare_traces(run_modes={"train", "eval"})
with self.assertRaises(KeyError):
est._warmup(warmup=True)
def instantiate_system():
system = sample_system_object()
model = fe.build(model_fn=fe.architecture.tensorflow.LeNet,
optimizer_fn='adam',
model_name='tf')
system.network = fe.Network(ops=[
fe.op.tensorop.meta.OneOf(
TestTensorOp(
inputs="x_out", outputs="x_out", mode="train", var=1),
TestTensorOp(
inputs="x_out", outputs="x_out", mode="train", var=1)),
ModelOp(model=model, inputs="x_out", outputs="y_pred")
])
return system
def test_saliency(self):
fe.estimator.enable_deterministic(200)
label_mapping = {
'airplane': 0,
'automobile': 1,
'bird': 2,
'cat': 3,
'deer': 4,
'dog': 5,
'frog': 6,
'horse': 7,
'ship': 8,
'truck': 9
}
batch_size = 32
train_data, eval_data = cifar10.load_data()
pipeline = fe.Pipeline(test_data=train_data,
batch_size=batch_size,
ops=[Normalize(inputs="x", outputs="x")],
num_process=0)
weight_path = os.path.abspath(os.path.join(__file__, "..", "resources", "lenet_cifar10_tf.h5"))
model = fe.build(model_fn=lambda: LeNet(input_shape=(32, 32, 3)), optimizer_fn="adam", weights_path=weight_path)
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred")
])
save_dir = tempfile.mkdtemp()
traces = [
Saliency(model=model,
model_inputs="x",
class_key="y",
model_outputs="y_pred",
samples=5,
label_mapping=label_mapping),
ImageSaver(inputs="saliency", save_dir=save_dir)
]
estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=5, traces=traces, log_steps=1000)
estimator.test()
ans_img_path = os.path.abspath(os.path.join(__file__, "..", "resources", "saliency_figure.png"))
ans_img = img_to_rgb_array(ans_img_path)
output_img_path = os.path.join(save_dir, "saliency_test_epoch_5.png")
output_img = img_to_rgb_array(output_img_path)
self.assertTrue(check_img_similar(output_img, ans_img))
def get_estimator(batch_size=100, epochs=20, max_train_steps_per_epoch=None, save_dir=tempfile.mkdtemp()):
train_data, _ = load_data()
pipeline = fe.Pipeline(
train_data=train_data,
batch_size=batch_size,
ops=[
ExpandDims(inputs="x", outputs="x", axis=0),
Minmax(inputs="x", outputs="x"),
Binarize(inputs="x", outputs="x", threshold=0.5),
])
encode_model = fe.build(model_fn=EncoderNet, optimizer_fn="adam", model_name="encoder")
decode_model = fe.build(model_fn=DecoderNet, optimizer_fn="adam", model_name="decoder")
network = fe.Network(ops=[
ModelOp(model=encode_model, inputs="x", outputs="meanlogvar"),
SplitOp(inputs="meanlogvar", outputs=("mean", "logvar")),
ReparameterizepOp(inputs=("mean", "logvar"), outputs="z"),
ModelOp(model=decode_model, inputs="z", outputs="x_logit"),
CrossEntropy(inputs=("x_logit", "x"), outputs="cross_entropy"),
CVAELoss(inputs=("cross_entropy", "mean", "logvar", "z"), outputs="loss"),
UpdateOp(model=encode_model, loss_name="loss"),
UpdateOp(model=decode_model, loss_name="loss"),
])
traces = [
BestModelSaver(model=encode_model, save_dir=save_dir), BestModelSaver(model=decode_model, save_dir=save_dir)
]
estimator = fe.Estimator(pipeline=pipeline,
network=network,
epochs=epochs,
traces=traces,
max_train_steps_per_epoch=max_train_steps_per_epoch)
return estimator
def test_tf_model_end_to_end_gradient(self):
train_data, _ = mnist.load_data()
pipeline = fe.Pipeline(train_data=train_data,
batch_size=4,
ops=[ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x")])
model = fe.build(model_fn=LeNet_tf, optimizer_fn="adam")
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce"),
GradientOp(model=model, finals="ce", outputs="gradients"),
UpdateOp(model=model, gradients="gradients", loss_name="ce")
])
estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2, max_train_steps_per_epoch=10)
estimator.fit()
def get_estimator(epochs=30,
batch_size=128,
seq_length=20,
vocab_size=10000,
data_dir=None,
train_steps_per_epoch=None,
save_dir=tempfile.mkdtemp()):
train_data, eval_data, _, _ = load_data(root_dir=data_dir,
seq_length=seq_length + 1)
pipeline = fe.Pipeline(train_data=train_data,
eval_data=eval_data,
ops=[
CreateInputAndTarget(inputs="x",
outputs=("x", "y")),
Batch(batch_size=batch_size, drop_last=True)
])
# step 2
model = fe.build(
model_fn=lambda: BuildModel(
vocab_size, embedding_dim=300, rnn_units=600),
optimizer_fn=lambda x: torch.optim.SGD(x, lr=1.0, momentum=0.9))
network = fe.Network(ops=[
DimesionAdjust(inputs=("x", "y"), outputs=("x", "y")),
ModelOp(model=model, inputs="x", outputs="y_pred", mode=None),
CrossEntropy(inputs=("y_pred", "y"),
outputs="ce",
form="sparse",
from_logits=True),
UpdateOp(model=model, loss_name="ce")
])
# step 3
traces = [
Perplexity(inputs="ce", outputs="perplexity", mode="eval"),
LRScheduler(model=model,
lr_fn=lambda step: lr_schedule(step, init_lr=1.0)),
BestModelSaver(model=model,
save_dir=save_dir,
metric='perplexity',
save_best_mode='min',
load_best_final=True),
EarlyStopping(monitor="perplexity", patience=5)
]
estimator = fe.Estimator(pipeline=pipeline,
network=network,
epochs=epochs,
traces=traces,
train_steps_per_epoch=train_steps_per_epoch)
return estimator
def get_estimator():
ds, _ = mnist.load_data()
ds = NegativeImageSimulatedTube(ds)
pipeline = fe.Pipeline(train_data=ds,
ops=[
ExpandDims(inputs="x", outputs="x"),
Minmax(inputs="x", outputs="x")
])
model = fe.build(model_fn=LeNet, optimizer_fn="adam")
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce"),
UpdateOp(model=model, loss_name="ce")
])
estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2)
return estimator
def get_estimator():
pos_real, _ = mnist.load_data()
neg_real, _ = mnist.load_data()
neg_sim, _ = mnist.load_data()
neg_sim = NegativeImageSimulatedTube(neg_sim)
batch_ds = BatchDataset(datasets=(pos_real, neg_real, neg_sim), num_samples=(2, 2, 1))
pipeline = fe.Pipeline(train_data=batch_ds,
ops=[ExpandDims(inputs="x", outputs="x"), Minmax(inputs="x", outputs="x")])
model = fe.build(model_fn=LeNet, optimizer_fn="adam")
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce"),
UpdateOp(model=model, loss_name="ce")
])
estimator = fe.Estimator(pipeline=pipeline, network=network, epochs=2)
return estimator
def test_network_collect_model_with_model_op_and_update_op(self):
model_fns = {"tf_model_fn": one_layer_tf_model, "torch_model_fn": OneLayerTorchModel}
for name, model_fn in model_fns.items():
with self.subTest(name):
model = fe.build(model_fn=model_fn, optimizer_fn=None)
model2 = fe.build(model_fn=model_fn, optimizer_fn=None)
ops = [
SampleTensorOp(inputs="x", outputs="x"),
ModelOp(model=model, inputs="x", outputs="y2"),
UpdateOp(model=model2, loss_name="ce")
]
models = fe.network._collect_models(ops)
ans = {model, model2}
self.assertEqual(models, ans)
