def test_tf2_profiler_by_time(tf2_profiler_config_parser_by_time, out_dir):
"""
This test executes a TF2 training script, enables detailed TF profiling by time, and
verifies the number of events.
"""
assert tf2_profiler_config_parser_by_time.profiling_enabled
hook = Hook(out_dir=out_dir)
helper_keras_fit(trial_dir=out_dir,
hook=hook,
eager=True,
steps=["train", "eval", "predict"])
hook.close()
# get tensorboard timeline files
files = []
for path in Path(tf2_profiler_config_parser_by_time.config.local_path +
"/framework").rglob(f"*{TENSORBOARDTIMELINE_SUFFIX}"):
files.append(path)
assert len(files) == 1
trace_file = str(files[0])
t_events = TensorboardProfilerEvents()
t_events.read_events_from_file(trace_file)
all_trace_events = t_events.get_all_events()
num_trace_events = len(all_trace_events)
print(f"Number of events read = {num_trace_events}")
# The number of events is varying by a small number on
# consecutive runs. Hence, the approximation in the below asserts.
assert num_trace_events >= 700
def helper_tensorflow_tests(use_keras, collection, save_config,
with_timestamp):
coll_name, coll_regex = collection
run_id = "trial_" + coll_name + "-" + datetime.now().strftime(
"%Y%m%d-%H%M%S%f")
trial_dir = os.path.join(SMDEBUG_TF_HOOK_TESTS_DIR, run_id)
if use_keras:
hook = TF_KerasHook(
out_dir=trial_dir,
include_collections=[coll_name],
save_config=save_config,
export_tensorboard=True,
)
saved_scalars = simple_tf_model(hook, with_timestamp=with_timestamp)
else:
hook = TF_SessionHook(
out_dir=trial_dir,
include_collections=[coll_name],
save_config=save_config,
export_tensorboard=True,
)
saved_scalars = tf_session_model(hook, with_timestamp=with_timestamp)
tf.reset_default_graph()
hook.close()
verify_files(trial_dir, save_config, saved_scalars)
if with_timestamp:
check_tf_events(trial_dir, saved_scalars)
def test_tf2_profiler_by_time(tf2_profiler_config_parser_by_time, out_dir):
"""
This test executes a TF2 training script, enables detailed TF profiling by time, and
verifies the number of events.
"""
assert tf2_profiler_config_parser_by_time.profiling_enabled
hook = Hook(out_dir=out_dir)
helper_keras_fit(trial_dir=out_dir,
hook=hook,
eager=True,
steps=["train", "eval", "predict"])
hook.close()
verify_detailed_profiling(out_dir, 700)
def helper_tensorflow_tests(use_keras, collection, save_config):
coll_name, coll_regex = collection
run_id = "trial_" + coll_name + "-" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
trial_dir = os.path.join(SMDEBUG_TF_HOOK_TESTS_DIR, run_id)
if use_keras:
hook = TF_KerasHook(
out_dir=trial_dir,
include_collections=[coll_name],
save_config=save_config,
export_tensorboard=True,
)
simple_tf_model(hook)
saved_scalars = [
"scalar/tf_keras_num_steps",
"scalar/tf_keras_before_train",
"scalar/tf_keras_after_train",
]
else:
hook = TF_SessionHook(
out_dir=trial_dir,
include_collections=[coll_name],
save_config=save_config,
export_tensorboard=True,
)
tf_session_model(hook)
tf.reset_default_graph()
saved_scalars = [
"scalar/tf_session_num_steps",
"scalar/tf_session_before_train",
"scalar/tf_session_after_train",
]
hook.close()
verify_files(trial_dir, save_config, saved_scalars)
def helper_tensorflow_tests(collection, save_config):
coll_name, coll_regex = collection
run_id = "trial_" + coll_name + "-" + datetime.now().strftime(
"%Y%m%d-%H%M%S%f")
trial_dir = os.path.join(SMDEBUG_TF_HOOK_TESTS_DIR, run_id)
hook = TF_Hook(out_dir=trial_dir,
include_collections=[coll_name],
export_tensorboard=True)
coll = hook.get_collection(coll_name)
coll.save_config = save_config
save_steps = save_config.get_save_config(ModeKeys.TRAIN).save_steps
if not save_steps:
save_interval = save_config.get_save_config(
ModeKeys.TRAIN).save_interval
save_steps = [i for i in range(0, 10, save_interval)]
simple_tf_model(hook)
hook.close()
saved_scalars = ["loss"]
check_trials(trial_dir, save_steps, coll_name, saved_scalars)
check_metrics_file(saved_scalars)