def build(self, epoch, checkpoint_manager): with TaskGroup(WorkspaceType.GLOBAL) as upload_task_group: for node, manager in checkpoint_manager._node_managers: with Node(str(node)), Task(): src_path = db_name(epoch, manager._node_name, manager._db_prefix) dest_path = os.path.join(self.dest_dir, str(node)) ops.Python((local_copy_op, [src_path, dest_path], {}))([], []) return upload_task_group
def test_multi_instance_python_op(self): """ When task instances are created at runtime, C++ concurrently creates multiple instances of operators in C++, and concurrently destroys them once the task is finished. This means that the destructor of PythonOp will be called concurrently, so the GIL must be acquired. This test exercises this condition. """ with Task(num_instances=64) as task: with ops.loop(4): ops.Python((python_op_builder, [], {}))([], []) with LocalSession() as session: PythonOpStats.num_instances = 0 PythonOpStats.num_calls = 0 session.run(task) self.assertEquals(PythonOpStats.num_instances, 64) self.assertEquals(PythonOpStats.num_calls, 256)