def get_gluon_random_data_run(log_models=True):
mlflow.gluon.autolog(log_models)
with mlflow.start_run() as run:
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
validation = DataLoader(LogsDataset(),
batch_size=128,
last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(
model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
},
)
est = get_estimator(model, trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3, val_data=validation)
client = mlflow.tracking.MlflowClient()
return client.get_run(run.info.run_id)
def test_autolog_persists_manually_created_run():
kiwi.gluon.autolog()
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
with kiwi.start_run() as run:
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": .001,
"epsilon": 1e-07
})
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
metrics=Accuracy(),
trainer=trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3)
assert kiwi.active_run().info.run_id == run.info.run_id
def test_autolog_registering_model():
registered_model_name = "test_autolog_registered_model"
mlflow.gluon.autolog(registered_model_name=registered_model_name)
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
})
est = get_estimator(model, trainer)
with mlflow.start_run(), warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3)
registered_model = MlflowClient().get_registered_model(
registered_model_name)
assert registered_model.name == registered_model_name
def test_autolog_ends_auto_created_run():
mlflow.gluon.autolog()
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
})
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
trainer=trainer,
**get_metrics())
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3)
assert mlflow.active_run() is None
def gluon_model(model_data):
train_data, train_label, _ = model_data
train_data_loader = DataLoader(list(zip(train_data, train_label)),
batch_size=128,
last_batch="discard")
model = HybridSequential()
model.add(Dense(128, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": .001,
"epsilon": 1e-07
})
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
metrics=Accuracy(),
trainer=trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(train_data_loader, epochs=3)
return model
def build_generator(n_filters, n_channels, mx_ctx):
netG = HybridSequential()
with netG.name_scope():
# Input is Z
netG.add(Conv2DTranspose(n_filters * 8, kernel_size=4, strides=1, padding=0, use_bias=False))
netG.add(BatchNorm())
netG.add(Activation("relu"))
netG.add(Conv2DTranspose(n_filters * 4, kernel_size=4, strides=2, padding=1, use_bias=False))
netG.add(BatchNorm())
netG.add(Activation("relu"))
netG.add(Conv2DTranspose(n_filters * 2, kernel_size=4, strides=2, padding=1, use_bias=False))
netG.add(BatchNorm())
netG.add(Activation("relu"))
netG.add(Conv2DTranspose(n_filters, kernel_size=4, strides=2, padding=1, use_bias=False))
netG.add(BatchNorm())
netG.add(Activation("relu"))
netG.add(Conv2DTranspose(n_channels, kernel_size=4, strides=2, padding=1, use_bias=False))
netG.add(BatchNorm())
netG.add(Activation("tanh"))
netG.initialize(mx.init.Normal(0.02), ctx=mx_ctx)
netG.hybridize()
return netG
def test_autolog_persists_manually_created_run():
mlflow.gluon.autolog()
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
with mlflow.start_run() as run:
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(
model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
},
)
est = get_estimator(model, trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3)
assert mlflow.active_run().info.run_id == run.info.run_id
def gluon_random_data_run():
mlflow.gluon.autolog()
with mlflow.start_run() as run:
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
validation = DataLoader(LogsDataset(),
batch_size=128,
last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": .001,
"epsilon": 1e-07
})
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
metrics=Accuracy(),
trainer=trainer)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(data, epochs=3, val_data=validation)
return client.get_run(run.info.run_id)
def gluon_model(model_data):
train_data, train_label, _ = model_data
train_data_loader = DataLoader(list(zip(train_data, train_label)),
batch_size=128,
last_batch="discard")
model = HybridSequential()
model.add(Dense(128, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(model.collect_params(),
"adam",
optimizer_params={
"learning_rate": 0.001,
"epsilon": 1e-07
})
# `metrics` was renamed in mxnet 1.6.0: https://github.com/apache/incubator-mxnet/pull/17048
arg_name = ("metrics"
if LooseVersion(mx.__version__) < LooseVersion("1.6.0") else
"train_metrics")
est = estimator.Estimator(net=model,
loss=SoftmaxCrossEntropyLoss(),
trainer=trainer,
**{arg_name: Accuracy()})
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(train_data_loader, epochs=3)
return model
def build_model(A, X):
model = HybridSequential()
hidden_layer_specs = [(4, 'tanh'), (2, 'tanh')]
in_units = in_units = X.shape[1]
with model.name_scope():
features, out_units = build_features(A, X)
model.add(features)
classifier = LogisticRegressor(out_units)
model.add(classifier)
model.hybridize()
model.initialize(Uniform(1))
return model, features
def build_model(A, X):
model = HybridSequential()
with model.name_scope():
features, out_units = build_features(A, X)
model.add(features)
logger.info("GCN Summary: \n{}".format(model))
classifier = LogisticRegressor(out_units)
model.add(classifier)
logger.info("GCN + LR Summary: \n{}".format(model))
model.hybridize()
model.initialize(Uniform(1))
return model, features
class CNN(HybridBlock):
def __init__(self, training=False, **kwargs):
super(CNN, self).__init__(**kwargs)
self.cnn = HybridSequential()
self.cnn.add( # We don't need pooling, since local information matters
Conv2D(channels=384, kernel_size=3, padding=1, activation='relu'), # Sees 3*3
Conv2D(channels=256, kernel_size=3, padding=1, activation='relu'), # Sees 5*5
Dense(units=1024, activation='relu'),
Dropout(0.2 if training else 0.0),
Dense(units=512, activation='relu'),
Dropout(0.2 if training else 0.0),
Dense(units=256, activation='relu'),
Dense(1))
self.cnn.hybridize()
def hybrid_forward(self, F, x):
return self.cnn(x)
def gluon_model(model_data):
train_data, train_label, _ = model_data
dataset = mx.gluon.data.ArrayDataset(train_data, train_label)
train_data_loader = DataLoader(dataset, batch_size=128, last_batch="discard")
model = HybridSequential()
model.add(Dense(128, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(
model.collect_params(), "adam", optimizer_params={"learning_rate": 0.001, "epsilon": 1e-07}
)
est = get_estimator(model, trainer)
est.fit(train_data_loader, epochs=3)
return model
def test_autolog_ends_auto_created_run():
mlflow.gluon.autolog()
data = DataLoader(LogsDataset(), batch_size=128, last_batch="discard")
model = HybridSequential()
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(10))
model.initialize()
model.hybridize()
trainer = Trainer(
model.collect_params(), "adam", optimizer_params={"learning_rate": 0.001, "epsilon": 1e-07}
)
est = get_estimator(model, trainer)
est.fit(data, epochs=3)
assert mlflow.active_run() is None
def __init__(self, transforms):
super(Compose, self).__init__()
transforms.append(None)
hybrid = []
for i in transforms:
if isinstance(i, HybridBlock):
hybrid.append(i)
continue
elif len(hybrid) == 1:
self.add(hybrid[0])
hybrid = []
elif len(hybrid) > 1:
hblock = HybridSequential()
for j in hybrid:
hblock.add(j)
hblock.hybridize()
self.add(hblock)
hybrid = []
if i is not None:
self.add(i)
def build_discriminator(n_filters, n_channels, mx_ctx):
netD = HybridSequential()
with netD.name_scope():
# Input is n_channels * 64 * 64
netD.add(Conv2D(n_filters, kernel_size=4, strides=2, padding=1, use_bias=False))
netD.add(LeakyReLU(0.2))
netD.add(Conv2D(n_filters * 2, kernel_size=4, strides=2, padding=1, use_bias=False))
netD.add(BatchNorm())
netD.add(LeakyReLU(0.2))
netD.add(Conv2D(n_filters * 4, kernel_size=4, strides=2, padding=1, use_bias=False))
netD.add(BatchNorm())
netD.add(LeakyReLU(0.2))
netD.add(Conv2D(n_filters * 8, kernel_size=4, strides=2, padding=1, use_bias=False))
netD.add(BatchNorm())
netD.add(LeakyReLU(0.2))
netD.add(Conv2D(1, 4, 1, 0, use_bias=False))
netD.initialize(mx.init.Normal(0.02), ctx=mx_ctx)
netD.hybridize()
num_workers=4)
mnist_valid = gluon.data.vision.FashionMNIST(train=False)
valid_data = gluon.data.DataLoader(mnist_valid.transform_first(transformer),
batch_size=batch_size,
num_workers=4)
# Only hybrid based networks can be exported
net = HybridSequential()
net.add(
Conv2D(channels=6, kernel_size=5, activation="relu"),
MaxPool2D(pool_size=2, strides=2),
Conv2D(channels=16, kernel_size=3, activation="relu"),
MaxPool2D(pool_size=2, strides=2),
Flatten(),
Dense(120, activation="relu"),
Dense(84, activation="relu"),
Dense(10),
)
net.initialize(init=init.Xavier())
# Only after hybridization a model can be exported with architecture included
net.hybridize()
trainer = Trainer(net.collect_params(), "sgd", {"learning_rate": 0.1})
est = estimator.Estimator(net=net,
loss=SoftmaxCrossEntropyLoss(),
train_metrics=Accuracy(),
trainer=trainer)
est.fit(train_data=train_data, epochs=2, val_data=valid_data)
def train(args: argparse.Namespace) -> HybridBlock:
session = boto3.session.Session()
client = session.client(service_name="secretsmanager",
region_name="us-east-1")
mlflow_secret = client.get_secret_value(SecretId=args.mlflow_secret)
mlflowdb_conf = json.loads(mlflow_secret["SecretString"])
converters.encoders[np.float64] = converters.escape_float
converters.conversions = converters.encoders.copy()
converters.conversions.update(converters.decoders)
mlflow.set_tracking_uri(
f"mysql+pymysql://{mlflowdb_conf['username']}:{mlflowdb_conf['password']}@{mlflowdb_conf['host']}/mlflow"
)
if mlflow.get_experiment_by_name(args.mlflow_experiment) is None:
mlflow.create_experiment(args.mlflow_experiment,
args.mlflow_artifacts_location)
mlflow.set_experiment(args.mlflow_experiment)
col_names = ["target"] + [f"kinematic_{i}" for i in range(1, 22)]
train_df = pd.read_csv(f"{args.train_channel}/train.csv.gz",
header=None,
names=col_names)
val_df = pd.read_csv(f"{args.validation_channel}/val.csv.gz",
header=None,
names=col_names)
train_X = train_df.drop("target", axis=1)
train_y = train_df["target"]
train_dataset = ArrayDataset(train_X.to_numpy(dtype="float32"),
train_y.to_numpy(dtype="float32"))
train = DataLoader(train_dataset, batch_size=args.batch_size)
val_X = val_df.drop("target", axis=1)
val_y = val_df["target"]
val_dataset = ArrayDataset(val_X.to_numpy(dtype="float32"),
val_y.to_numpy(dtype="float32"))
validation = DataLoader(val_dataset, batch_size=args.batch_size)
ctx = [gpu(i) for i in range(args.gpus)] if args.gpus > 0 else cpu()
mlflow.gluon.autolog()
with mlflow.start_run():
net = HybridSequential()
with net.name_scope():
net.add(Dense(256))
net.add(Dropout(.2))
net.add(Dense(64))
net.add(Dropout(.1))
net.add(Dense(16))
net.add(Dense(2))
net.initialize(Xavier(magnitude=2.24), ctx=ctx)
net.hybridize()
trainer = Trainer(net.collect_params(), "sgd",
{"learning_rate": args.learning_rate})
est = estimator.Estimator(net=net,
loss=SoftmaxCrossEntropyLoss(),
trainer=trainer,
train_metrics=Accuracy(),
context=ctx)
est.fit(train, epochs=args.epochs, val_data=validation)
return net
def export_block_for_cplusplus(path=None, block=None, data_shape=None, epoch=0, preprocess=True,
layout='HWC',
ctx=mx.cpu(), remove_amp_cast=True):
"""Helper function to export a HybridBlock to symbol JSON to be used by
`SymbolBlock.imports`, `mxnet.mod.Module` or the C++ interface..
Parameters
----------
path : str
Path to save model.
Two files path-symbol.json and path-xxxx.params will be created,
where xxxx is the 4 digits epoch number.
block : mxnet.gluon.HybridBlock
The hybridizable block. Note that normal gluon.Block is not supported.
data_shape : tuple of int, default is None
Fake data shape just for export purpose, in format (H, W, C).
If you don't specify ``data_shape``, `export_block` will try use some common data_shapes,
e.g., (224, 224, 3), (256, 256, 3), (299, 299, 3), (512, 512, 3)...
If any of this ``data_shape`` goes through, the export will succeed.
epoch : int
Epoch number of saved model.
preprocess : mxnet.gluon.HybridBlock, default is True.
Preprocess block prior to the network.
By default (True), it will subtract mean [123.675, 116.28, 103.53], divide
std [58.395, 57.12, 57.375], and convert original image (B, H, W, C and range [0, 255]) to
tensor (B, C, H, W) as network input. This is the default preprocess behavior of all GluonCV
pre-trained models.
You can use custom pre-process hybrid block or disable by set ``preprocess=None``.
layout : str, default is 'HWC'
The layout for raw input data. By default is HWC. Supports 'HWC' and 'CHW'.
Note that image channel order is always RGB.
ctx: mx.Context, default mx.cpu()
Network context.
Returns
-------
preprocessed block
"""
# input image layout
if data_shape is None:
data_shapes = [(s, s, 3) for s in (224, 256, 299, 300, 320, 416, 512, 600)]
else:
data_shapes = [data_shape]
'''
c++ 에서 inference 할 때,
데이터를 0 ~ 1 범위로 바꾸고, mxnet 입력형태인 (1, C, H, W)로 바꿀 필요가 없다.
그대로
'''
if preprocess:
# add preprocess block
if preprocess is True:
preprocess = _DefaultPreprocess()
else:
if not isinstance(preprocess, HybridBlock):
raise TypeError("preprocess must be HybridBlock, given {}".format(type(preprocess)))
wrapper_block = HybridSequential()
preprocess.initialize(ctx=ctx)
wrapper_block.add(preprocess)
wrapper_block.add(block)
else:
wrapper_block = block
wrapper_block.collect_params().reset_ctx(ctx)
# try different data_shape if possible, until one fits the network
last_exception = None
for dshape in data_shapes:
h, w, c = dshape
if layout == 'HWC': # 보통 HWC(opencv)형식이다.
x = mx.nd.zeros((1, h, w, c), ctx=ctx)
elif layout == 'CHW':
x = mx.nd.zeros((1, c, h, w), ctx=ctx)
# hybridize and forward once
wrapper_block.hybridize()
try:
wrapper_block(x)
if path != None:
wrapper_block.export(path, epoch, remove_amp_cast=remove_amp_cast)
last_exception = None
break
except MXNetError as e:
last_exception = e
if last_exception is not None:
raise RuntimeError(str(last_exception).splitlines()[0])
return wrapper_block
