def create_hook(output_s3_uri):
# Create a SaveConfig that determines tensors from which steps are to be stored.
# With the following SaveConfig, we will save tensors for steps 1, 2 and 3.
save_config = SaveConfig(save_steps=[1, 2, 3])
# Create a hook that logs all the tensors seen while training the model.
hook = Hook(out_dir=output_s3_uri, save_config=save_config, save_all=True)
return hook
def test_hook_custom_collection():
save_config = SaveConfig(save_steps=[0, 1, 2, 3])
run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
out_dir = "/tmp/" + run_id
hook = t_hook(out_dir=out_dir, save_config=save_config, include_collections=["ReluActivation"])
hook.get_collection("ReluActivation").include(["relu*", "input_*"])
run_mnist_gluon_model(hook=hook, num_steps_train=10, num_steps_eval=10)
shutil.rmtree(out_dir)
def create_hook(output_uri, save_frequency):
# With the following SaveConfig, we will save tensors with the save_interval 100.
save_config = SaveConfig(save_interval=save_frequency)
# Create a hook that logs weights, biases and gradients while training the model.
hook = Hook(
out_dir=output_uri,
save_config=save_config,
include_collections=["weights", "gradients", "biases"],
)
return hook
def test_save_config(hook=None):
if hook is None:
save_config_collection = SaveConfig(save_steps=[4, 5, 6])
run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
out_dir = "/tmp/" + run_id
save_config = SaveConfig(save_steps=[0, 1, 2, 3])
hook = t_hook(
out_dir=out_dir,
save_config=save_config,
include_collections=[
"ReluActivation", "weights", "biases", "gradients", "default"
],
)
custom_collect = hook.get_collection("ReluActivation")
custom_collect.save_config = save_config_collection
custom_collect.include(["relu*", "input_*", "output*"])
run_mnist_gluon_model(hook=hook, num_steps_train=10, num_steps_eval=10)
if hook is None:
shutil.rmtree(out_dir)
def create_hook(output_s3_uri):
# With the following SaveConfig, we will save tensors for steps 1, 2 and 3
# (indexing starts with 0).
save_config = SaveConfig(save_steps=[1, 2, 3])
# Create a hook that logs weights, biases and gradients while training the model.
hook = Hook(
out_dir=output_s3_uri,
save_config=save_config,
include_collections=["weights", "gradients", "biases"],
)
return hook
def test_save_shapes(out_dir):
global_reduce_config = ReductionConfig(save_shape=True)
global_save_config = SaveConfig(save_steps=[0, 1])
hook = t_hook(
out_dir=out_dir,
save_config=global_save_config,
save_all=True,
reduction_config=global_reduce_config,
)
run_mnist_gluon_model(hook=hook, num_steps_train=5)
verify_shapes(out_dir, 0)
verify_shapes(out_dir, 1)
shutil.rmtree(out_dir)
def create_hook(output_s3_uri, block):
# Create a SaveConfig that determines tensors from which steps are to be stored.
# With the following SaveConfig, we will save tensors for steps 1, 2 and 3.
save_config = SaveConfig(save_steps=[1, 2, 3])
# Create a hook that logs weights, biases, gradients and inputs outputs of model while training.
hook = Hook(
out_dir=output_s3_uri,
save_config=save_config,
include_collections=["weights", "gradients", "biases", "TopBlock"],
)
# The names of input and output tensors of a block are in following format
# Inputs : <block_name>_input_<input_index>, and
# Output : <block_name>_output
# In order to log the inputs and output of a model, we will create a collection as follows:
hook.get_collection("TopBlock").add_block_tensors(block, inputs=True, outputs=True)
return hook
def test_save_all(hook=None, out_dir=None):
hook_created = False
if hook is None:
hook_created = True
save_config = SaveConfig(save_steps=[0, 1, 2, 3])
run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
out_dir = "/tmp/" + run_id
print("Registering the hook with out_dir {}".format(out_dir))
hook = t_hook(out_dir=out_dir, save_config=save_config, save_all=True)
run_mnist_gluon_model(hook=hook, num_steps_train=7, num_steps_eval=5)
# assert for steps and tensor_names
print("Created the trial with out_dir {}".format(out_dir))
tr = create_trial(out_dir)
tensor_list = tr.tensor_names()
assert tr
assert len(tr.steps()) == 4
# some tensor names, like input and output, can't be retrieved from training session, so here we only assert for tensor numbers
# 46 is gotten from index file
# if no assertion failure, then the script could save all tensors
assert len(tensor_list) == 46
if hook_created:
shutil.rmtree(out_dir)
def test_modes(hook=None, path=None):
if hook is None:
run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
path = "/tmp/" + run_id
hook = t_hook(
out_dir=path,
save_config=SaveConfig({
modes.TRAIN: SaveConfigMode(save_interval=2),
modes.EVAL: SaveConfigMode(save_interval=3),
}),
include_collections=["gradients", "weights"],
)
run_mnist_gluon_model(hook=hook,
set_modes=True,
register_to_loss_block=True,
num_steps_train=6,
num_steps_eval=6)
tr = create_trial(path)
assert len(tr.modes()) == 2
assert len(tr.steps()) == 5, tr.steps()
assert len(tr.steps(mode=modes.TRAIN)) == 3
assert len(tr.steps(mode=modes.EVAL)) == 2, tr.steps()
# Ensure that the gradients are available in TRAIN modes only.
grad_tns_name = tr.tensor_names(regex="^gradient.")[0]
grad_tns = tr.tensor(grad_tns_name)
grad_train_steps = grad_tns.steps(mode=modes.TRAIN)
grad_eval_steps = grad_tns.steps(mode=modes.EVAL)
assert len(grad_train_steps) == 3
assert grad_eval_steps == []
# Ensure that the weights are available in TRAIN and EVAL modes.
wt_tns_name = tr.tensor_names(regex="conv\d+_weight")[0]
wt_tns = tr.tensor(wt_tns_name)
wt_train_steps = wt_tns.steps(mode=modes.TRAIN)
wt_eval_steps = wt_tns.steps(mode=modes.EVAL)
assert len(wt_train_steps) == 3
assert len(wt_eval_steps) == 2
def test_save_config(hook=None, out_dir=None):
hook_created = False
if hook is None:
hook_created = True
global_reduce_config = ReductionConfig(reductions=["max", "mean"])
global_save_config = SaveConfig(save_steps=[0, 1, 2, 3])
run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
out_dir = "/tmp/newlogsRunTest/" + run_id
print("Registering the hook with out_dir {0}".format(out_dir))
hook = t_hook(
out_dir=out_dir,
save_config=global_save_config,
save_all=True,
include_collections=[
"weights",
"biases",
"gradients",
"default",
"ReluActivation",
"flatten",
],
reduction_config=global_reduce_config,
)
hook.get_collection("ReluActivation").include(["relu*"])
hook.get_collection("ReluActivation").save_config = SaveConfig(
save_steps=[4, 5, 6])
hook.get_collection(
"ReluActivation").reduction_config = ReductionConfig(
reductions=["min"], abs_reductions=["max"])
hook.get_collection("flatten").include(["flatten*"])
hook.get_collection("flatten").save_config = SaveConfig(
save_steps=[4, 5, 6])
hook.get_collection("flatten").reduction_config = ReductionConfig(
norms=["l1"], abs_norms=["l2"])
run_mnist_gluon_model(hook=hook, num_steps_train=10, num_steps_eval=10)
# Testing
print("Created the trial with out_dir {0}".format(out_dir))
tr = create_trial(out_dir)
assert tr
assert len(tr.steps()) == 7
print(tr.tensor_names())
tname = tr.tensor_names(regex=r"conv\d+_weight")[0]
# Global reduction with max and mean
weight_tensor = tr.tensor(tname)
max_val = weight_tensor.reduction_value(step_num=1,
abs=False,
reduction_name="max")
assert max_val is not None
mean_val = weight_tensor.reduction_value(step_num=1,
abs=False,
reduction_name="mean")
assert mean_val is not None
# custom reduction at step 4 with reduction = 'min' and abs reduction = 'max'
tname = tr.tensor_names(regex=r"conv\d+_relu_input_0")[0]
relu_input = tr.tensor(tname)
min_val = relu_input.reduction_value(step_num=4,
abs=False,
reduction_name="min")
assert min_val is not None
abs_max_val = relu_input.reduction_value(step_num=4,
abs=True,
reduction_name="max")
assert abs_max_val is not None
# Custom reduction with normalization
tname = tr.tensor_names(regex=r"flatten\d+_input_0")[0]
flatten_input = tr.tensor(tname)
l1_norm = flatten_input.reduction_value(step_num=4,
abs=False,
reduction_name="l1")
assert l1_norm is not None
l2_norm = flatten_input.reduction_value(step_num=4,
abs=True,
reduction_name="l2")
assert l2_norm is not None
if hook_created:
shutil.rmtree(out_dir)