def build(self, epoch, checkpoint_manager):
     with TaskGroup(WorkspaceType.GLOBAL) as upload_task_group:
         for node, manager in checkpoint_manager._node_managers:
             with Node(str(node)), Task():
                 src_path = db_name(epoch, manager._node_name, manager._db_prefix)
                 dest_path = os.path.join(self.dest_dir, str(node))
                 ops.Python((local_copy_op,
                             [src_path, dest_path], {}))([], [])
     return upload_task_group
Exemple #2
0
 def test_multi_instance_python_op(self):
     """
     When task instances are created at runtime, C++ concurrently creates
     multiple instances of operators in C++, and concurrently destroys them
     once the task is finished. This means that the destructor of PythonOp
     will be called concurrently, so the GIL must be acquired. This
     test exercises this condition.
     """
     with Task(num_instances=64) as task:
         with ops.loop(4):
             ops.Python((python_op_builder, [], {}))([], [])
     with LocalSession() as session:
         PythonOpStats.num_instances = 0
         PythonOpStats.num_calls = 0
         session.run(task)
         self.assertEquals(PythonOpStats.num_instances, 64)
         self.assertEquals(PythonOpStats.num_calls, 256)