def test_find_faces_to_represent_surface_regular_wrapper( small_grid_and_surface: Tuple[RegularGrid, Surface]): # Arrange grid, surface = small_grid_and_surface grid_epc = surface_epc = grid.model.epc_file grid_uuid = grid.uuid surface_uuid = surface.uuid name = "test" input_index = 0 use_index_as_realisation = False # Act index, success, epc_file, uuid_list = find_faces_to_represent_surface_regular_wrapper( input_index, use_index_as_realisation, grid_epc, grid_uuid, surface_epc, surface_uuid, name) model = Model(epc_file=epc_file) rm_tree("tmp_dir") # Assert assert success is True assert index == input_index assert len(model.uuids(obj_type='LocalDepth3dCrs')) == 1 assert len(model.uuids(obj_type='IjkGridRepresentation')) == 1 assert len(model.uuids(obj_type='TriangulatedSetRepresentation')) == 1 assert len(model.uuids(obj_type='GridConnectionSetRepresentation')) == 1 assert len(model.uuids(obj_type='FaultInterpretation')) == 1 assert len(model.uuids(obj_type='TectonicBoundaryFeature')) == 1 assert len(model.uuids()) == 6 assert len(uuid_list) == 3
def function_multiprocessing( function: Callable, kwargs_list: List[Dict[str, Any]], recombined_epc: Union[Path, str], cluster, consolidate: bool = True, ) -> List[bool]: """Calls a function concurrently with the specfied arguments. A multiprocessing pool is used to call the function multiple times in parallel. Once all results are returned, they are combined into a single epc file. Args: function (Callable): the function to be called. Needs to return: - index (int): the index of the kwargs in the kwargs_list. - success (bool): whether the function call was successful, whatever that definiton is. - epc_file (Path/str): the epc file path where the objects are stored. - uuid_list (List[str]): list of UUIDs of relevant objects. kwargs_list (List[Dict[Any]]): A list of keyword argument dictionaries that are used when calling the function. recombined_epc (Path/str): A pathlib Path or path string of where the combined epc will be saved. cluster (LocalCluster/JobQueueCluster): a LocalCluster is a Dask cluster on a local machine. If using a job queing system, a JobQueueCluster can be used such as an SGECluster, SLURMCluster, PBSCluster, LSFCluster etc. consolidate (bool): if True and an equivalent part already exists in a model, it is not duplicated and the uuids are noted as equivalent. Returns: success_list (List[bool]): A boolean list of successful function calls. Note: This function uses the Dask backend to run the given function in parallel, so a Dask cluster must be setup and passed as an argument. Dask will need to be installed in the Python environment because it is not a dependency of the project. More info can be found at https://docs.dask.org/en/latest/deploying.html """ log.info("Multiprocessing function called with %s function.", function.__name__) for i, kwargs in enumerate(kwargs_list): kwargs["index"] = i with parallel_backend("dask"): results = Parallel()(delayed(function)(**kwargs) for kwargs in kwargs_list) log.info("Function calls complete.") # Sorting the results by the original kwargs_list index. results = list(sorted(results, key = lambda x: x[0])) success_list = [result[1] for result in results] epc_list = [result[2] for result in results] uuids_list = [result[3] for result in results] log.info("Number of successes: %s/%s.", sum(success_list), len(results)) epc_file = Path(str(recombined_epc)) if epc_file.is_file(): model_recombined = Model(epc_file = str(epc_file)) else: model_recombined = new_model(epc_file = str(epc_file)) log.info("Creating the recombined epc file.") for i, epc in enumerate(epc_list): if epc is None: continue while True: try: model = Model(epc_file = epc) break except FileNotFoundError: time.sleep(1) continue uuids = uuids_list[i] if uuids is None: uuids = model.uuids() for uuid in uuids: model_recombined.copy_uuid_from_other_model(model, uuid = uuid, consolidate = consolidate) # Deleting temporary directory. log.info("Deleting the temporary directory") rm_tree("tmp_dir") model_recombined.store_epc() log.info("Recombined epc file complete.") return success_list