def testAddDebugTensorWatches_defaultDebugOp(self): debug_utils.add_debug_tensor_watch(self._run_options, "foo/node_a", 1, debug_urls="file:///tmp/tfdbg_1") debug_utils.add_debug_tensor_watch(self._run_options, "foo/node_b", 0, debug_urls="file:///tmp/tfdbg_2") debug_watch_opts = self._run_options.debug_options.debug_tensor_watch_opts self.assertEqual(2, len(debug_watch_opts)) watch_0 = debug_watch_opts[0] watch_1 = debug_watch_opts[1] self.assertEqual("foo/node_a", watch_0.node_name) self.assertEqual(1, watch_0.output_slot) self.assertEqual("foo/node_b", watch_1.node_name) self.assertEqual(0, watch_1.output_slot) # Verify default debug op name. self.assertEqual(["DebugIdentity"], watch_0.debug_ops) self.assertEqual(["DebugIdentity"], watch_1.debug_ops) # Verify debug URLs. self.assertEqual(["file:///tmp/tfdbg_1"], watch_0.debug_urls) self.assertEqual(["file:///tmp/tfdbg_2"], watch_1.debug_urls)
def testGetGrpcDebugWatchesReturnsCorrectAnswer(self): with session.Session() as sess: v = variables.Variable(50.0, name="v") delta = constant_op.constant(5.0, name="delta") inc_v = state_ops.assign_add(v, delta, name="inc_v") sess.run(v.initializer) # Before any debugged runs, the server should be aware of no debug # watches. self.assertEqual([], self._server_1.gated_grpc_debug_watches()) run_metadata = config_pb2.RunMetadata() run_options = config_pb2.RunOptions(output_partition_graphs=True) debug_utils.add_debug_tensor_watch( run_options, "delta", output_slot=0, debug_ops=["DebugNumericSummary(gated_grpc=true)"], debug_urls=[self._debug_server_url_1]) debug_utils.add_debug_tensor_watch( run_options, "v", output_slot=0, debug_ops=["DebugIdentity"], debug_urls=[self._debug_server_url_1]) sess.run(inc_v, options=run_options, run_metadata=run_metadata) # After the first run, the server should have noted the debug watches # for which gated_grpc == True, but not the ones with gated_grpc == False. self.assertEqual(1, len(self._server_1.gated_grpc_debug_watches())) debug_watch = self._server_1.gated_grpc_debug_watches()[0] self.assertEqual("delta", debug_watch.node_name) self.assertEqual(0, debug_watch.output_slot) self.assertEqual("DebugNumericSummary", debug_watch.debug_op)
def testGetGrpcDebugWatchesReturnsCorrectAnswer(self): with session.Session() as sess: v = variables.Variable(50.0, name="v") delta = constant_op.constant(5.0, name="delta") inc_v = state_ops.assign_add(v, delta, name="inc_v") sess.run(v.initializer) # Before any debugged runs, the server should be aware of no debug # watches. self.assertEqual([], self._server_1.gated_grpc_debug_watches()) run_metadata = config_pb2.RunMetadata() run_options = config_pb2.RunOptions(output_partition_graphs=True) debug_utils.add_debug_tensor_watch( run_options, "delta", output_slot=0, debug_ops=["DebugNumericSummary(gated_grpc=true)"], debug_urls=[self._debug_server_url_1]) debug_utils.add_debug_tensor_watch( run_options, "v", output_slot=0, debug_ops=["DebugIdentity"], debug_urls=[self._debug_server_url_1]) sess.run(inc_v, options=run_options, run_metadata=run_metadata) # After the first run, the server should have noted the debug watches # for which gated_grpc == True, but not the ones with gated_grpc == False. self.assertEqual(1, len(self._server_1.gated_grpc_debug_watches())) debug_watch = self._server_1.gated_grpc_debug_watches()[0] self.assertEqual("delta", debug_watch.node_name) self.assertEqual(0, debug_watch.output_slot) self.assertEqual("DebugNumericSummary", debug_watch.debug_op)
def testDumpUninitializedVariable(self): op_namespace = "testDumpUninitializedVariable" with session.Session() as sess: u_init_val = np.array([[5.0, 3.0], [-1.0, 0.0]]) s_init_val = b"str1" u_name = "%s/u" % op_namespace s_name = "%s/s" % op_namespace u_init = constant_op.constant(u_init_val, shape=[2, 2]) u = variables.Variable(u_init, name=u_name) s_init = constant_op.constant(s_init_val) s = variables.Variable(s_init, name=s_name) run_options = config_pb2.RunOptions(output_partition_graphs=True) debug_urls = self._debug_urls() # Add debug tensor watch for u. debug_utils.add_debug_tensor_watch(run_options, "%s" % u_name, 0, debug_urls=debug_urls) debug_utils.add_debug_tensor_watch(run_options, "%s" % s_name, 0, debug_urls=debug_urls) run_metadata = config_pb2.RunMetadata() # Initialize u and s. sess.run(variables.global_variables_initializer(), options=run_options, run_metadata=run_metadata) # Verify the dump file for the uninitialized value of u. dump = debug_data.DebugDumpDir( self._dump_root, partition_graphs=run_metadata.partition_graphs) self.assertEqual(2, dump.size) self.assertEqual(self._expected_partition_graph_count, len(run_metadata.partition_graphs)) # Verify that the variable is properly initialized by the run() call. u_vals = dump.get_tensors(u_name, 0, "DebugIdentity") s_vals = dump.get_tensors(s_name, 0, "DebugIdentity") self.assertEqual(1, len(u_vals)) self.assertIsNone(u_vals[0]) self.assertEqual(1, len(s_vals)) self.assertIsNone(s_vals[0]) # Call run() again, to check that u is initialized properly. self.assertAllClose(u_init_val, sess.run(u)) self.assertEqual(s_init_val, sess.run(s))
def _generate_dump_from_simple_addition_graph(self): with session.Session() as sess: u_init_val = np.array([[5.0, 3.0], [-1.0, 0.0]]) v_init_val = np.array([[2.0], [-1.0]]) # Use node names with overlapping namespace (i.e., parent directory) to # test concurrent, non-racing directory creation. u_name = "u" v_name = "v" w_name = "w" u_init = constant_op.constant(u_init_val, shape=[2, 2]) u = variables.Variable(u_init, name=u_name) v_init = constant_op.constant(v_init_val, shape=[2, 1]) v = variables.Variable(v_init, name=v_name) w = math_ops.matmul(u, v, name=w_name) u.initializer.run() v.initializer.run() run_options = config_pb2.RunOptions(output_partition_graphs=True) debug_urls = "file://%s" % self._dump_root # Add debug tensor watch for u. debug_utils.add_debug_tensor_watch(run_options, "%s/read" % u_name, 0, debug_urls=debug_urls) # Add debug tensor watch for v. debug_utils.add_debug_tensor_watch(run_options, "%s/read" % v_name, 0, debug_urls=debug_urls) run_metadata = config_pb2.RunMetadata() # Invoke Session.run(). sess.run(w, options=run_options, run_metadata=run_metadata) self.assertEqual(self._expected_partition_graph_count, len(run_metadata.partition_graphs)) dump = debug_data.DebugDumpDir( self._dump_root, partition_graphs=run_metadata.partition_graphs) simple_add_results = collections.namedtuple("SimpleAddResults", [ "u_init_val", "v_init_val", "u", "v", "w", "u_name", "v_name", "w_name", "dump" ]) return simple_add_results(u_init_val, v_init_val, u, v, w, u_name, v_name, w_name, dump)
def testAddDebugTensorWatches_explicitDebugOp(self): debug_utils.add_debug_tensor_watch(self._run_options, "foo/node_a", 0, debug_ops="DebugNanCount", debug_urls="file:///tmp/tfdbg_1") debug_watch_opts = self._run_options.debug_options.debug_tensor_watch_opts self.assertEqual(1, len(debug_watch_opts)) watch_0 = debug_watch_opts[0] self.assertEqual("foo/node_a", watch_0.node_name) self.assertEqual(0, watch_0.output_slot) # Verify default debug op name. self.assertEqual(["DebugNanCount"], watch_0.debug_ops) # Verify debug URLs. self.assertEqual(["file:///tmp/tfdbg_1"], watch_0.debug_urls)
def _prepare_cont_call_dump_path_and_run_options(self): """Prepare the dump path and RunOptions for next cont() call. Returns: dump_path: (str) Directory path to which the intermediate tensor will be dumped. run_options: (config_pb2.RunOptions) The RunOptions containing the tensor watch options for this graph. """ run_options = config_pb2.RunOptions() dump_path = self._cont_call_dump_path() for element_name in self._closure_elements: if ":" in element_name: debug_utils.add_debug_tensor_watch( run_options, debug_graphs.get_node_name(element_name), output_slot=debug_graphs.get_output_slot(element_name), debug_urls=["file://" + dump_path]) return dump_path, run_options
def _prepare_cont_call_dump_path_and_run_options(self): """Prepare the dump path and RunOptions for next cont() call. Returns: dump_path: (str) Directory path to which the intermediate tensor will be dumped. run_options: (config_pb2.RunOptions) The RunOptions containing the tensor watch options for this graph. """ run_options = config_pb2.RunOptions() dump_path = self._cont_call_dump_path() for element_name in self._closure_elements: if ":" in element_name: debug_utils.add_debug_tensor_watch( run_options, debug_data.get_node_name(element_name), output_slot=debug_data.get_output_slot(element_name), debug_urls=["file://" + dump_path]) return dump_path, run_options
def testAddDebugTensorWatches_multipleDebugOps(self): debug_utils.add_debug_tensor_watch( self._run_options, "foo/node_a", 0, debug_ops=["DebugNanCount", "DebugIdentity"], debug_urls="file:///tmp/tfdbg_1") debug_watch_opts = self._run_options.debug_options.debug_tensor_watch_opts self.assertEqual(1, len(debug_watch_opts)) watch_0 = debug_watch_opts[0] self.assertEqual("foo/node_a", watch_0.node_name) self.assertEqual(0, watch_0.output_slot) # Verify default debug op name. self.assertEqual(["DebugNanCount", "DebugIdentity"], watch_0.debug_ops) # Verify debug URLs. self.assertEqual(["file:///tmp/tfdbg_1"], watch_0.debug_urls)
def testAddDebugTensorWatches_defaultDebugOp(self): debug_utils.add_debug_tensor_watch( self._run_options, "foo/node_a", 1, debug_urls="file:///tmp/tfdbg_1") debug_utils.add_debug_tensor_watch( self._run_options, "foo/node_b", 0, debug_urls="file:///tmp/tfdbg_2") debug_watch_opts = self._run_options.debug_options.debug_tensor_watch_opts self.assertEqual(2, len(debug_watch_opts)) watch_0 = debug_watch_opts[0] watch_1 = debug_watch_opts[1] self.assertEqual("foo/node_a", watch_0.node_name) self.assertEqual(1, watch_0.output_slot) self.assertEqual("foo/node_b", watch_1.node_name) self.assertEqual(0, watch_1.output_slot) # Verify default debug op name. self.assertEqual(["DebugIdentity"], watch_0.debug_ops) self.assertEqual(["DebugIdentity"], watch_1.debug_ops) # Verify debug URLs. self.assertEqual(["file:///tmp/tfdbg_1"], watch_0.debug_urls) self.assertEqual(["file:///tmp/tfdbg_2"], watch_1.debug_urls)
def testWatchingOnlyOneOfTwoOutputSlotsDoesNotLeadToCausalityFailure(self): with session.Session() as sess: x_name = "oneOfTwoSlots/x" u_name = "oneOfTwoSlots/u" v_name = "oneOfTwoSlots/v" w_name = "oneOfTwoSlots/w" y_name = "oneOfTwoSlots/y" x = variables.Variable([1, 3, 3, 7], dtype=dtypes.int32, name=x_name) sess.run(x.initializer) unique_x, indices, _ = array_ops.unique_with_counts(x, name=u_name) v = math_ops.add(unique_x, unique_x, name=v_name) w = math_ops.add(indices, indices, name=w_name) y = math_ops.add(w, w, name=y_name) run_options = config_pb2.RunOptions(output_partition_graphs=True) # Watch only the first output slot of u, even though it has two output # slots. debug_utils.add_debug_tensor_watch(run_options, u_name, 0, debug_urls=self._debug_urls()) debug_utils.add_debug_tensor_watch(run_options, w_name, 0, debug_urls=self._debug_urls()) debug_utils.add_debug_tensor_watch(run_options, y_name, 0, debug_urls=self._debug_urls()) run_metadata = config_pb2.RunMetadata() sess.run([v, y], options=run_options, run_metadata=run_metadata) dump = debug_data.DebugDumpDir( self._dump_root, partition_graphs=run_metadata.partition_graphs, validate=True) self.assertAllClose([1, 3, 7], dump.get_tensors(u_name, 0, "DebugIdentity")[0])
def testAllowsDifferentWatchesOnDifferentRuns(self): """Test watching different tensors on different runs of the same graph.""" with session.Session(config=self._no_rewrite_session_config()) as sess: u_init_val = [[5.0, 3.0], [-1.0, 0.0]] v_init_val = [[2.0], [-1.0]] # Use node names with overlapping namespace (i.e., parent directory) to # test concurrent, non-racing directory creation. u_name = "diff_Watch/u" v_name = "diff_Watch/v" u_init = constant_op.constant(u_init_val, shape=[2, 2]) u = variables.Variable(u_init, name=u_name) v_init = constant_op.constant(v_init_val, shape=[2, 1]) v = variables.Variable(v_init, name=v_name) w = math_ops.matmul(u, v, name="diff_Watch/matmul") u.initializer.run() v.initializer.run() for i in range(2): run_options = config_pb2.RunOptions(output_partition_graphs=True) run_dump_root = self._debug_dump_dir(run_number=i) debug_urls = self._debug_urls(run_number=i) if i == 0: # First debug run: Add debug tensor watch for u. debug_utils.add_debug_tensor_watch( run_options, "%s/read" % u_name, 0, debug_urls=debug_urls) else: # Second debug run: Add debug tensor watch for v. debug_utils.add_debug_tensor_watch( run_options, "%s/read" % v_name, 0, debug_urls=debug_urls) run_metadata = config_pb2.RunMetadata() # Invoke Session.run(). sess.run(w, options=run_options, run_metadata=run_metadata) self.assertEqual(self._expected_partition_graph_count, len(run_metadata.partition_graphs)) dump = debug_data.DebugDumpDir( run_dump_root, partition_graphs=run_metadata.partition_graphs) self.assertTrue(dump.loaded_partition_graphs()) # Each run should have generated only one dumped tensor, not two. self.assertEqual(1, dump.size) if i == 0: self.assertAllClose([u_init_val], dump.get_tensors("%s/read" % u_name, 0, "DebugIdentity")) self.assertGreaterEqual( dump.get_rel_timestamps("%s/read" % u_name, 0, "DebugIdentity")[0], 0) else: self.assertAllClose([v_init_val], dump.get_tensors("%s/read" % v_name, 0, "DebugIdentity")) self.assertGreaterEqual( dump.get_rel_timestamps("%s/read" % v_name, 0, "DebugIdentity")[0], 0)
def _poll_server_till_success(max_attempts, sleep_per_poll_sec, debug_server_url, dump_dir, server, gpu_memory_fraction=1.0): """Poll server until success or exceeding max polling count. Args: max_attempts: (int) How many times to poll at maximum sleep_per_poll_sec: (float) How many seconds to sleep for after each unsuccessful poll. debug_server_url: (str) gRPC URL to the debug server. dump_dir: (str) Dump directory to look for files in. If None, will directly check data from the server object. server: The server object. gpu_memory_fraction: (float) Fraction of GPU memory to be allocated for the Session used in server polling. Returns: (bool) Whether the polling succeeded within max_polls attempts. """ poll_count = 0 config = config_pb2.ConfigProto(gpu_options=config_pb2.GPUOptions( per_process_gpu_memory_fraction=gpu_memory_fraction)) with session.Session(config=config) as sess: for poll_count in range(max_attempts): server.clear_data() print("Polling: poll_count = %d" % poll_count) x_init_name = "x_init_%d" % poll_count x_init = constant_op.constant([42.0], shape=[1], name=x_init_name) x = variables.Variable(x_init, name=x_init_name) run_options = config_pb2.RunOptions() debug_utils.add_debug_tensor_watch( run_options, x_init_name, 0, debug_urls=[debug_server_url]) try: sess.run(x.initializer, options=run_options) except errors.FailedPreconditionError: pass if dump_dir: if os.path.isdir( dump_dir) and debug_data.DebugDumpDir(dump_dir).size > 0: shutil.rmtree(dump_dir) print("Poll succeeded.") return True else: print("Poll failed. Sleeping for %f s" % sleep_per_poll_sec) time.sleep(sleep_per_poll_sec) else: if server.debug_tensor_values: print("Poll succeeded.") return True else: print("Poll failed. Sleeping for %f s" % sleep_per_poll_sec) time.sleep(sleep_per_poll_sec) return False
def _poll_server_till_success(max_attempts, sleep_per_poll_sec, debug_server_url, dump_dir, server, gpu_memory_fraction=1.0): """Poll server until success or exceeding max polling count. Args: max_attempts: (int) How many times to poll at maximum sleep_per_poll_sec: (float) How many seconds to sleep for after each unsuccessful poll. debug_server_url: (str) gRPC URL to the debug server. dump_dir: (str) Dump directory to look for files in. If None, will directly check data from the server object. server: The server object. gpu_memory_fraction: (float) Fraction of GPU memory to be allocated for the Session used in server polling. Returns: (bool) Whether the polling succeeded within max_polls attempts. """ poll_count = 0 config = config_pb2.ConfigProto(gpu_options=config_pb2.GPUOptions( per_process_gpu_memory_fraction=gpu_memory_fraction)) with session.Session(config=config) as sess: for poll_count in range(max_attempts): server.clear_data() print("Polling: poll_count = %d" % poll_count) x_init_name = "x_init_%d" % poll_count x_init = constant_op.constant([42.0], shape=[1], name=x_init_name) x = variables.Variable(x_init, name=x_init_name) run_options = config_pb2.RunOptions() debug_utils.add_debug_tensor_watch( run_options, x_init_name, 0, debug_urls=[debug_server_url]) try: sess.run(x.initializer, options=run_options) except errors.FailedPreconditionError: pass if dump_dir: if os.path.isdir( dump_dir) and debug_data.DebugDumpDir(dump_dir).size > 0: shutil.rmtree(dump_dir) print("Poll succeeded.") return True else: print("Poll failed. Sleeping for %f s" % sleep_per_poll_sec) time.sleep(sleep_per_poll_sec) else: if server.debug_tensor_values: print("Poll succeeded.") return True else: print("Poll failed. Sleeping for %f s" % sleep_per_poll_sec) time.sleep(sleep_per_poll_sec) return False
def testDebugWhileLoopGeneratesMultipleDumps(self): with session.Session() as sess: num_iter = 10 # "u" is the Variable being updated in the loop. u_name = "testDumpToFileWhileLoop/u" u_namespace = u_name.split("/")[0] u_init_val = np.array(11.0) u_init = constant_op.constant(u_init_val) u = variables.Variable(u_init, name=u_name) # "v" is the increment. v_name = "testDumpToFileWhileLoop/v" v_namespace = v_name.split("/")[0] v_init_val = np.array(2.0) v_init = constant_op.constant(v_init_val) v = variables.Variable(v_init, name=v_name) u.initializer.run() v.initializer.run() i = constant_op.constant(0, name="testDumpToFileWhileLoop/i") def cond(i): return math_ops.less(i, num_iter) def body(i): new_u = state_ops.assign_add(u, v) new_i = math_ops.add(i, 1) op = control_flow_ops.group(new_u) new_i = control_flow_ops.with_dependencies([op], new_i) return [new_i] loop = control_flow_ops.while_loop(cond, body, [i], parallel_iterations=1) # Create RunOptions for debug-watching tensors run_options = config_pb2.RunOptions(output_partition_graphs=True) debug_urls = self._debug_urls() # Add debug tensor watch for u. debug_utils.add_debug_tensor_watch(run_options, u_name, 0, debug_urls=debug_urls) # Add debug tensor watch for v. debug_utils.add_debug_tensor_watch(run_options, "%s/read" % v_name, 0, debug_urls=debug_urls) # Add debug tensor watch for while/Identity. debug_utils.add_debug_tensor_watch(run_options, "while/Identity", 0, debug_urls=debug_urls) # Add debug tensor watch for while/Add/y. debug_utils.add_debug_tensor_watch(run_options, "while/Add/y", 0, debug_urls=debug_urls) run_metadata = config_pb2.RunMetadata() r = sess.run(loop, options=run_options, run_metadata=run_metadata) self.assertEqual(self._expected_partition_graph_count, len(run_metadata.partition_graphs)) self.assertEqual(num_iter, r) u_val_final = sess.run(u) self.assertAllClose(u_init_val + num_iter * v_init_val, u_val_final) # Verify dump files self.assertTrue(os.path.isdir(self._dump_root)) self.assertTrue( os.path.isdir(os.path.join(self._dump_root, u_namespace))) self.assertTrue( os.path.isdir(os.path.join(self._dump_root, v_namespace, "v"))) dump = debug_data.DebugDumpDir( self._dump_root, partition_graphs=run_metadata.partition_graphs) # Expected dumped tensors: u, v/read, 10 iterations of while/Identity, # and 10 iterations of while/Add/y. self.assertEqual(1 + 1 + num_iter + num_iter, dump.size) # Verify tensor values. self.assertAllClose([u_init_val], dump.get_tensors(u_name, 0, "DebugIdentity")) self.assertAllClose([v_init_val], dump.get_tensors("%s/read" % v_name, 0, "DebugIdentity")) while_id_tensors = dump.get_tensors("while/Identity", 0, "DebugIdentity") self.assertEqual(10, len(while_id_tensors)) for k in xrange(len(while_id_tensors)): self.assertAllClose(np.array(k), while_id_tensors[k]) # Verify ascending timestamps from the while loops. while_id_rel_timestamps = dump.get_rel_timestamps( "while/Identity", 0, "DebugIdentity") while_id_dump_sizes_bytes = dump.get_dump_sizes_bytes( "while/Identity", 0, "DebugIdentity") self.assertEqual(10, len(while_id_rel_timestamps)) prev_rel_time = 0 prev_dump_size_bytes = while_id_dump_sizes_bytes[0] for rel_time, dump_size_bytes in zip(while_id_rel_timestamps, while_id_dump_sizes_bytes): self.assertGreaterEqual(rel_time, prev_rel_time) self.assertEqual(dump_size_bytes, prev_dump_size_bytes) prev_rel_time = rel_time prev_dump_size_bytes = dump_size_bytes # Test querying debug watch keys from node name. watch_keys = dump.debug_watch_keys("while/Identity") self.assertEqual(["while/Identity:0:DebugIdentity"], watch_keys) # Test querying debug datum instances from debug watch key. self.assertEqual(10, len(dump.watch_key_to_data(watch_keys[0]))) self.assertEqual([], dump.watch_key_to_data("foo"))
def testDumpStringTensorsWorks(self): with session.Session() as sess: str1_init_val = np.array(b"abc") str2_init_val = np.array(b"def") str1_init = constant_op.constant(str1_init_val) str2_init = constant_op.constant(str2_init_val) str1_name = "str1" str2_name = "str2" str1 = variables.Variable(str1_init, name=str1_name) str2 = variables.Variable(str2_init, name=str2_name) # Concatenate str1 and str2 str_concat = math_ops.add(str1, str2, name="str_concat") str1.initializer.run() str2.initializer.run() run_options = config_pb2.RunOptions(output_partition_graphs=True) debug_urls = self._debug_urls() # Add debug tensor watch for u. debug_utils.add_debug_tensor_watch(run_options, "%s/read" % str1_name, 0, debug_urls=debug_urls) # Add debug tensor watch for v. debug_utils.add_debug_tensor_watch(run_options, "%s/read" % str2_name, 0, debug_urls=debug_urls) run_metadata = config_pb2.RunMetadata() sess.run(str_concat, options=run_options, run_metadata=run_metadata) # String ops are located on CPU. self.assertEqual(1, len(run_metadata.partition_graphs)) dump = debug_data.DebugDumpDir( self._dump_root, partition_graphs=run_metadata.partition_graphs) self.assertIn(str1_name, dump.nodes()) self.assertIn(str2_name, dump.nodes()) self.assertEqual(2, dump.size) self.assertEqual([str1_init_val], dump.get_tensors("%s/read" % str1_name, 0, "DebugIdentity")) self.assertEqual([str2_init_val], dump.get_tensors("%s/read" % str2_name, 0, "DebugIdentity")) self.assertGreaterEqual( dump.get_rel_timestamps("%s/read" % str1_name, 0, "DebugIdentity")[0], 0) self.assertGreaterEqual( dump.get_rel_timestamps("%s/read" % str2_name, 0, "DebugIdentity")[0], 0) self.assertGreater( dump.get_dump_sizes_bytes("%s/read" % str1_name, 0, "DebugIdentity")[0], 0) self.assertGreater( dump.get_dump_sizes_bytes("%s/read" % str2_name, 0, "DebugIdentity")[0], 0)