df_train = MapDataComponent(df_train, one_hot, 1)
df_val = MapDataComponent(df_val, one_hot, 1)
M = KerasModel(
resnet50,
inputs_desc=[InputDesc(tf.uint8, [None, 224, 224, 3], 'images')],
targets_desc=[InputDesc(tf.float32, [None, 1000], 'labels')],
input=df_train,
trainer=SyncMultiGPUTrainerReplicated(num_gpu))
lr = tf.get_variable('learning_rate', initializer=0.1, trainable=False)
tf.summary.scalar('lr', lr)
M.compile(optimizer=tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True),
loss='categorical_crossentropy',
metrics='categorical_accuracy')
callbacks = [
ModelSaver(),
ScheduledHyperParamSetter('learning_rate', [(0, 0.1), (3, BASE_LR)],
interp='linear'), # warmup
ScheduledHyperParamSetter('learning_rate', [(30, BASE_LR * 0.1),
(60, BASE_LR * 1e-2),
(85, BASE_LR * 1e-3)]),
GPUUtilizationTracker()
]
if not args.fake:
callbacks.append(
DataParallelInferenceRunner(df_val,
ScalarStats(['categorical_accuracy']),
M.add(KL.Conv2D(32, 3, padding='same', activation='relu'))
M.add(KL.Flatten())
M.add(
KL.Dense(512,
activation='relu',
kernel_regularizer=keras.regularizers.l2(1e-5)))
M.add(KL.Dropout(0.5))
M.add(
KL.Dense(10,
activation=None,
kernel_regularizer=keras.regularizers.l2(1e-5)))
M.add(KL.Activation('softmax'))
return M
dataset_train, dataset_test = get_data()
M = KerasModel(model_func,
inputs_desc=[
InputDesc(tf.float32, [None, IMAGE_SIZE, IMAGE_SIZE, 1],
'images')
],
targets_desc=[InputDesc(tf.float32, [None, 10], 'labels')],
input=QueueInput(dataset_train))
M.compile(optimizer=tf.train.AdamOptimizer(1e-3),
loss='categorical_crossentropy',
metrics='categorical_accuracy')
M.fit(validation_data=dataset_test,
steps_per_epoch=dataset_train.size(),
callbacks=[ModelSaver()])
M.add(KL.Conv2D(32, 3, activation='relu', padding='same'))
M.add(KL.MaxPooling2D())
M.add(KL.Conv2D(32, 3, padding='same', activation='relu'))
M.add(KL.Flatten())
M.add(KL.Dense(512, activation='relu', kernel_regularizer=keras.regularizers.l2(1e-5)))
M.add(KL.Dropout(0.5))
M.add(KL.Dense(10, activation=None, kernel_regularizer=keras.regularizers.l2(1e-5)))
M.add(KL.Activation('softmax'))
return M
dataset_train, dataset_test = get_data()
M = KerasModel(
model_func,
inputs_desc=[InputDesc(tf.float32, [None, IMAGE_SIZE, IMAGE_SIZE, 1], 'images')],
targets_desc=[InputDesc(tf.float32, [None, 10], 'labels')],
input=QueueInput(dataset_train))
M.compile(
optimizer=tf.train.AdamOptimizer(1e-3),
loss='categorical_crossentropy',
metrics='categorical_accuracy'
)
M.fit(
validation_data=dataset_test,
steps_per_epoch=dataset_train.size(),
callbacks=[
ModelSaver()
]
)
val_set = dataflow.MapDataComponent(val_set, one_hot, 1)
###start building models
model_test_GPU_new = KerasModel(
build_model_resnet50,
inputs_desc=[
InputDesc(tf.float32, (None, ) + img_shape, 'images')
],
targets_desc=[
InputDesc(tf.float32, (None, len(target_name)), 'labels')
],
input=train_gen,
trainer=SyncMultiGPUTrainerParameterServer(num_GPU))
model_test_GPU_new.compile(optimizer=opt_use,
loss='categorical_crossentropy',
metrics='categorical_accuracy')
callbacks = [
ModelSaver(checkpoint_dir=checkpoint_dir),
# MinSaver('val-error-top1'), ##backup the model with best validation error
GPUUtilizationTracker(
), ## record GPU utilizations during training
ScheduledHyperParamSetter('learning_rate',
[(0, min(START_LR, BASE_LR)),
(10, BASE_LR * 1e-1),
(60, BASE_LR * 1e-2),
(90, BASE_LR * 1e-3),
(100, BASE_LR * 1e-4)]),
DataParallelInferenceRunner(
val_set,
ScalarStats(
def train(checkpoint_dir,
model_name,
dataset,
num_epochs,
quant_type,
batch_size_per_gpu,
lr=None,
post_quantize_only=False):
train_data, test_data, (img_shape,
label_shape) = datasets.DATASETS[dataset]()
num_gpus = max(gpu.get_num_gpu(), 1)
effective_batch_size = batch_size_per_gpu * num_gpus
train_data = BatchData(train_data, batch_size_per_gpu)
test_data = BatchData(test_data, batch_size_per_gpu, remainder=True)
steps_per_epoch = len(train_data) // num_gpus
if lr:
if isinstance(lr, str):
lr = ast.literal_eval(lr)
if isinstance(lr, float):
lr_schedule = [(0, lr)]
else:
lr_schedule = lr
else:
lr_schedule = [(0, 0.005), (8, 0.1), (25, 0.005), (30, 0)]
if num_epochs is None:
num_epochs = lr_schedule[-1][0]
if post_quantize_only:
start_quantising_at_epoch = 0
else:
start_quantising_at_epoch = lr_schedule[-2][0] if len(
lr_schedule) > 1 else max(0, num_epochs - 5)
logger.info(f"Training with LR schedule: {str(lr_schedule)}")
logger.info(f"Quantising at epoch {start_quantising_at_epoch}")
# train_data = FakeData([(batch_size_per_gpu,) + img_shape, (batch_size_per_gpu, ) + label_shape])
model_func, input_spec, output_spec = get_model_func(
"train",
model_name,
quant_type,
img_shape,
num_classes=label_shape[0],
quant_delay=steps_per_epoch * start_quantising_at_epoch)
target_spec = [
tf.TensorSpec(t.shape, t.dtype, name=t.name.split("/")[-1] + "_target")
for t in output_spec
]
model = KerasModel(get_model=model_func,
input_signature=input_spec,
target_signature=target_spec,
input=train_data,
trainer=SyncMultiGPUTrainerParameterServer(
num_gpus, ps_device='gpu'))
lr = tf.get_variable('learning_rate',
initializer=lr_schedule[0][1],
trainable=False)
tf.summary.scalar('learning_rate-summary', lr)
model.compile(optimizer=tf.train.MomentumOptimizer(learning_rate=lr,
momentum=0.9),
loss="categorical_crossentropy",
metrics=["categorical_accuracy"])
model.fit(steps_per_epoch=steps_per_epoch,
max_epoch=num_epochs,
callbacks=[
ModelSaver(max_to_keep=1, checkpoint_dir=checkpoint_dir),
DataParallelInferenceRunner(
test_data, ScalarStats(model._stats_to_inference),
num_gpus),
ScheduledHyperParamSetter('learning_rate',
lr_schedule,
interp="linear"),
StatMonitorParamSetter('learning_rate',
'validation_categorical_accuracy',
lambda x: x / 2,
threshold=0.001,
last_k=10,
reverse=True)
],
session_init=SaverRestore(checkpoint_dir + "/checkpoint")
if post_quantize_only else None)
df_train = MapDataComponent(df_train, one_hot, 1)
df_val = MapDataComponent(df_val, one_hot, 1)
M = KerasModel(
resnet50,
inputs_desc=[InputDesc(tf.uint8, [None, 224, 224, 3], 'images')],
targets_desc=[InputDesc(tf.float32, [None, 1000], 'labels')],
input=df_train,
trainer=SyncMultiGPUTrainerReplicated(nr_gpu))
lr = tf.get_variable('learning_rate', initializer=0.1, trainable=False)
tf.summary.scalar('lr', lr)
M.compile(
optimizer=tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True),
loss='categorical_crossentropy',
metrics='categorical_accuracy'
)
callbacks = [
ModelSaver(),
ScheduledHyperParamSetter(
'learning_rate',
[(0, 0.1), (3, BASE_LR)], interp='linear'), # warmup
ScheduledHyperParamSetter(
'learning_rate',
[(30, BASE_LR * 0.1), (60, BASE_LR * 1e-2), (85, BASE_LR * 1e-3)]),
GPUUtilizationTracker()
]
if not args.fake:
callbacks.append(
M.add(KL.Conv2D(32, 3, activation='relu', padding='same'))
M.add(KL.MaxPooling2D())
M.add(KL.Conv2D(32, 3, padding='same', activation='relu'))
M.add(KL.Flatten())
M.add(KL.Dense(512, activation='relu', kernel_regularizer=keras.regularizers.l2(1e-5)))
M.add(KL.Dropout(0.5))
M.add(KL.Dense(10, activation=None, kernel_regularizer=keras.regularizers.l2(1e-5)))
M.add(KL.Activation('softmax'))
return M
dataset_train, dataset_test = get_data()
M = KerasModel(
model_func,
input_signature=[tf.TensorSpec([None, IMAGE_SIZE, IMAGE_SIZE, 1], tf.float32, 'images')],
target_signature=[tf.TensorSpec([None, 10], tf.float32, 'labels')],
input=QueueInput(dataset_train))
M.compile(
optimizer=tf.optimizers.Adam(),
loss='categorical_crossentropy',
metrics='categorical_accuracy'
)
M.fit(
validation_data=dataset_test,
steps_per_epoch=len(dataset_train),
callbacks=[
ModelSaver()
]
)