def create_hook(output_s3_uri):
# With the following SaveConfig, we will save tensors for steps 0, 1, 2 and 3
# (indexing starts with 0).
save_config = SaveConfig(save_steps=[0, 1, 2, 3])
# Create a hook that logs weights, biases and gradients while training the model.
hook = Hook(
out_dir=output_s3_uri,
save_config=save_config,
include_collections=["ReluActivation", "weights", "biases", "gradients"],
)
hook.get_collection("ReluActivation").include(["relu*", "input_*"])
return hook
def helper_mxnet_tests(collection, register_loss, save_config):
coll_name, coll_regex = collection
run_id = "trial_" + coll_name + "-" + datetime.now().strftime(
"%Y%m%d-%H%M%S%f")
trial_dir = os.path.join(SMDEBUG_MX_HOOK_TESTS_DIR, run_id)
hook = MX_Hook(out_dir=trial_dir,
include_collections=[coll_name],
export_tensorboard=True)
coll = hook.get_collection(coll_name)
coll.save_config = save_config
save_steps = save_config.get_save_config(ModeKeys.TRAIN).save_steps
if not save_steps:
save_interval = save_config.get_save_config(
ModeKeys.TRAIN).save_interval
save_steps = [i for i in range(0, 10, save_interval)]
simple_mx_model(hook, register_loss=register_loss)
hook.close()
saved_scalars = [
"scalar/mx_before_train", "scalar/mx_train_loss",
"scalar/mx_after_train"
]
check_trials(trial_dir, save_steps, coll_name, saved_scalars)
check_metrics_file(saved_scalars)
def create_hook(output_s3_uri, block):
# Create a SaveConfig that determines tensors from which steps are to be stored.
# With the following SaveConfig, we will save tensors for steps 1, 2 and 3.
save_config = SaveConfig(save_steps=[1, 2, 3])
# Create a hook that logs weights, biases, gradients and inputs outputs of model while training.
hook = Hook(
out_dir=output_s3_uri,
save_config=save_config,
include_collections=["weights", "gradients", "biases", "TopBlock"],
)
# The names of input and output tensors of a block are in following format
# Inputs : <block_name>_input_<input_index>, and
# Output : <block_name>_output
# In order to log the inputs and output of a model, we will create a collection as follows:
hook.get_collection("TopBlock").add_block_tensors(block, inputs=True, outputs=True)
return hook