def test_autolog_registering_model():
registered_model_name = "test_autolog_registered_model"
mlflow.gluon.autolog(registered_model_name=registered_model_name)
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
})
est = get_estimator(model, trainer)
with mlflow.start_run(), warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3)
registered_model = MlflowClient().get_registered_model(
registered_model_name)
assert registered_model.name == registered_model_name
def get_gluon_random_data_run(log_models=True):
mlflow.gluon.autolog(log_models)
with mlflow.start_run() as run:
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
validation = DataLoader(LogsDataset(),
batch_size=128,
last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(
model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
},
)
est = get_estimator(model, trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3, val_data=validation)
client = mlflow.tracking.MlflowClient()
return client.get_run(run.info.run_id)
def gluon_model(model_data):
train_data, train_label, _ = model_data
train_data_loader = DataLoader(list(zip(train_data, train_label)),
batch_size=128,
last_batch="discard")
model = HybridSequential()
model.add(Dense(128, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": .001,
"epsilon": 1e-07
})
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
metrics=Accuracy(),
trainer=trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(train_data_loader, epochs=3)
return model
def test_autolog_persists_manually_created_run():
mlflow.gluon.autolog()
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
with mlflow.start_run() as run:
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(
model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
},
)
est = get_estimator(model, trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3)
assert mlflow.active_run().info.run_id == run.info.run_id
def test_autolog_ends_auto_created_run():
mlflow.gluon.autolog()
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
})
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
trainer=trainer,
**get_metrics())
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3)
assert mlflow.active_run() is None
def gluon_model(model_data):
train_data, train_label, _ = model_data
train_data_loader = DataLoader(list(zip(train_data, train_label)),
batch_size=128,
last_batch="discard")
model = HybridSequential()
model.add(Dense(128, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
})
# `metrics` was renamed in mxnet 1.6.0: https://github.com/apache/incubator-mxnet/pull/17048
arg_name = ("metrics"
if LooseVersion(mx.__version__) < LooseVersion("1.6.0") else
"train_metrics")
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
trainer=trainer,
**{arg_name: Accuracy()})
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(train_data_loader, epochs=3)
return model
def gluon_random_data_run():
mlflow.gluon.autolog()
with mlflow.start_run() as run:
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
validation = DataLoader(LogsDataset(),
batch_size=128,
last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": .001,
"epsilon": 1e-07
})
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
metrics=Accuracy(),
trainer=trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3, val_data=validation)
return client.get_run(run.info.run_id)
def test_autolog_persists_manually_created_run():
kiwi.gluon.autolog()
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
with kiwi.start_run() as run:
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": .001,
"epsilon": 1e-07
})
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
metrics=Accuracy(),
trainer=trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3)
assert kiwi.active_run().info.run_id == run.info.run_id
def gluon_model(model_data):
train_data, train_label, _ = model_data
dataset = mx.gluon.data.ArrayDataset(train_data, train_label)
train_data_loader = DataLoader(dataset, batch_size=128, last_batch="discard")
model = HybridSequential()
model.add(Dense(128, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(
model.collect_params(), "adam", optimizer_params={"learning_rate": 0.001, "epsilon": 1e-07}
)
est = get_estimator(model, trainer)
est.fit(train_data_loader, epochs=3)
return model
def test_autolog_ends_auto_created_run():
mlflow.gluon.autolog()
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(
model.collect_params(), "adam", optimizer_params={"learning_rate": 0.001, "epsilon": 1e-07}
)
est = get_estimator(model, trainer)
est.fit(data, epochs=3)
assert mlflow.active_run() is None
activation='relu'),
BatchNorm(axis=1, momentum=0.9),
MaxPool2D(pool_size=(2, 2), strides=(2, 2)),
# layer 5
Flatten(),
Dropout(0.3),
Dense(128, activation='relu'),
# layer 6
Dense(10))
# %%
# -- Initialize parameters
net.initialize(init=init.Xavier(), ctx=mx_ctx)
for name, param in net.collect_params().items():
print(name)
# %%
# -- Define loss function and optimizer
loss_fn = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'Adam', {'learning_rate': 0.001})
# %%%
# -- Custom metric function
def acc(output, label):
return (output.argmax(axis=1) == label.astype('float32')).mean().asscalar()
def export_block_for_cplusplus(path=None, block=None, data_shape=None, epoch=0, preprocess=True,
layout='HWC',
ctx=mx.cpu(), remove_amp_cast=True):
"""Helper function to export a HybridBlock to symbol JSON to be used by
`SymbolBlock.imports`, `mxnet.mod.Module` or the C++ interface..
Parameters
----------
path : str
Path to save model.
Two files path-symbol.json and path-xxxx.params will be created,
where xxxx is the 4 digits epoch number.
block : mxnet.gluon.HybridBlock
The hybridizable block. Note that normal gluon.Block is not supported.
data_shape : tuple of int, default is None
Fake data shape just for export purpose, in format (H, W, C).
If you don't specify ``data_shape``, `export_block` will try use some common data_shapes,
e.g., (224, 224, 3), (256, 256, 3), (299, 299, 3), (512, 512, 3)...
If any of this ``data_shape`` goes through, the export will succeed.
epoch : int
Epoch number of saved model.
preprocess : mxnet.gluon.HybridBlock, default is True.
Preprocess block prior to the network.
By default (True), it will subtract mean [123.675, 116.28, 103.53], divide
std [58.395, 57.12, 57.375], and convert original image (B, H, W, C and range [0, 255]) to
tensor (B, C, H, W) as network input. This is the default preprocess behavior of all GluonCV
pre-trained models.
You can use custom pre-process hybrid block or disable by set ``preprocess=None``.
layout : str, default is 'HWC'
The layout for raw input data. By default is HWC. Supports 'HWC' and 'CHW'.
Note that image channel order is always RGB.
ctx: mx.Context, default mx.cpu()
Network context.
Returns
-------
preprocessed block
"""
# input image layout
if data_shape is None:
data_shapes = [(s, s, 3) for s in (224, 256, 299, 300, 320, 416, 512, 600)]
else:
data_shapes = [data_shape]
'''
c++ 에서 inference 할 때,
데이터를 0 ~ 1 범위로 바꾸고, mxnet 입력형태인 (1, C, H, W)로 바꿀 필요가 없다.
그대로
'''
if preprocess:
# add preprocess block
if preprocess is True:
preprocess = _DefaultPreprocess()
else:
if not isinstance(preprocess, HybridBlock):
raise TypeError("preprocess must be HybridBlock, given {}".format(type(preprocess)))
wrapper_block = HybridSequential()
preprocess.initialize(ctx=ctx)
wrapper_block.add(preprocess)
wrapper_block.add(block)
else:
wrapper_block = block
wrapper_block.collect_params().reset_ctx(ctx)
# try different data_shape if possible, until one fits the network
last_exception = None
for dshape in data_shapes:
h, w, c = dshape
if layout == 'HWC': # 보통 HWC(opencv)형식이다.
x = mx.nd.zeros((1, h, w, c), ctx=ctx)
elif layout == 'CHW':
x = mx.nd.zeros((1, c, h, w), ctx=ctx)
# hybridize and forward once
wrapper_block.hybridize()
try:
wrapper_block(x)
if path != None:
wrapper_block.export(path, epoch, remove_amp_cast=remove_amp_cast)
last_exception = None
break
except MXNetError as e:
last_exception = e
if last_exception is not None:
raise RuntimeError(str(last_exception).splitlines()[0])
return wrapper_block
num_workers=4)
mnist_valid = gluon.data.vision.FashionMNIST(train=False)
valid_data = gluon.data.DataLoader(mnist_valid.transform_first(transformer),
batch_size=batch_size,
num_workers=4)
# Only hybrid based networks can be exported
net = HybridSequential()
net.add(
Conv2D(channels=6, kernel_size=5, activation="relu"),
MaxPool2D(pool_size=2, strides=2),
Conv2D(channels=16, kernel_size=3, activation="relu"),
MaxPool2D(pool_size=2, strides=2),
Flatten(),
Dense(120, activation="relu"),
Dense(84, activation="relu"),
Dense(10),
)
net.initialize(init=init.Xavier())
# Only after hybridization a model can be exported with architecture included
net.hybridize()
trainer = Trainer(net.collect_params(), "sgd", {"learning_rate": 0.1})
est = estimator.Estimator(net=net,
loss=SoftmaxCrossEntropyLoss(),
train_metrics=Accuracy(),
trainer=trainer)
est.fit(train_data=train_data, epochs=2, val_data=valid_data)
def train(args: argparse.Namespace) -> HybridBlock:
session = boto3.session.Session()
client = session.client(service_name="secretsmanager",
region_name="us-east-1")
mlflow_secret = client.get_secret_value(SecretId=args.mlflow_secret)
mlflowdb_conf = json.loads(mlflow_secret["SecretString"])
converters.encoders[np.float64] = converters.escape_float
converters.conversions = converters.encoders.copy()
converters.conversions.update(converters.decoders)
mlflow.set_tracking_uri(
f"mysql+pymysql://{mlflowdb_conf['username']}:{mlflowdb_conf['password']}@{mlflowdb_conf['host']}/mlflow"
)
if mlflow.get_experiment_by_name(args.mlflow_experiment) is None:
mlflow.create_experiment(args.mlflow_experiment,
args.mlflow_artifacts_location)
mlflow.set_experiment(args.mlflow_experiment)
col_names = ["target"] + [f"kinematic_{i}" for i in range(1, 22)]
train_df = pd.read_csv(f"{args.train_channel}/train.csv.gz",
header=None,
names=col_names)
val_df = pd.read_csv(f"{args.validation_channel}/val.csv.gz",
header=None,
names=col_names)
train_X = train_df.drop("target", axis=1)
train_y = train_df["target"]
train_dataset = ArrayDataset(train_X.to_numpy(dtype="float32"),
train_y.to_numpy(dtype="float32"))
train = DataLoader(train_dataset, batch_size=args.batch_size)
val_X = val_df.drop("target", axis=1)
val_y = val_df["target"]
val_dataset = ArrayDataset(val_X.to_numpy(dtype="float32"),
val_y.to_numpy(dtype="float32"))
validation = DataLoader(val_dataset, batch_size=args.batch_size)
ctx = [gpu(i) for i in range(args.gpus)] if args.gpus > 0 else cpu()
mlflow.gluon.autolog()
with mlflow.start_run():
net = HybridSequential()
with net.name_scope():
net.add(Dense(256))
net.add(Dropout(.2))
net.add(Dense(64))
net.add(Dropout(.1))
net.add(Dense(16))
net.add(Dense(2))
net.initialize(Xavier(magnitude=2.24), ctx=ctx)
net.hybridize()
trainer = Trainer(net.collect_params(), "sgd",
{"learning_rate": args.learning_rate})
est = estimator.Estimator(net=net,
loss=SoftmaxCrossEntropyLoss(),
trainer=trainer,
train_metrics=Accuracy(),
context=ctx)
est.fit(train, epochs=args.epochs, val_data=validation)
return net
return d_iter
batch_size = 256
train_iter, test_iter = d2l.load_data_mnist(batch_size=1)
train_iter = blendData(train_iter, imgs)
test_iter = blendData(test_iter, imgs)
num_classes = 2
net = HybridSequential()
net.add(Conv2D(channels=3, kernel_size=5, activation='sigmoid'),
MaxPool2D(pool_size=2, strides=2),
Conv2D(channels=8, kernel_size=5, activation='sigmoid'),
MaxPool2D(pool_size=2, strides=2),
Conv2D(channels=120, kernel_size=4, activation='sigmoid'),
Conv2D(channels=84, kernel_size=1, activation='sigmoid'),
Conv2D(channels=10, kernel_size=1),
Conv2DTranspose(num_classes, kernel_size=56, padding=14, strides=28, activation='sigmoid'))
lr, num_epochs = 0.9, 10
ctx = d2l.try_gpu()
net.initialize(force_reinit=True, ctx=ctx, init=init.Xavier())
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr})
d2l.train_ch5(net, train_iter, test_iter, batch_size, trainer, ctx, num_epochs)
# %%
# -- Hybridize and run a forward pass once to generate a symbol which will be used later
# for plotting the network.
net.hybridize()
net(x)
# %%
# -- Define loss function and optimizer
loss_fn = gluon.loss.SoftmaxCrossEntropyLoss()
lr_scheduler = mx.lr_scheduler.FactorScheduler(base_lr=0.001,
factor=0.333,
step=10 * len(train_data))
trainer = gluon.Trainer(net.collect_params(), 'Adam',
{'lr_scheduler': lr_scheduler})
train_metric = mx.metric.Accuracy()
eval_metric = mx.metric.Accuracy()
# %%%
# -- Custom metric function
def acc(output, label):
return (output.argmax(axis=1) == label.astype('float32')).mean().asscalar()
# %%
# -- Train
