def chunked_or_distributed(items: Iterable[Any], max_groups: int, optimal_group_size: int) -> Iterable[Iterable[Any]]: """Divide *items* into at most *max_groups*. If possible, produces fewer than *max_groups*, but with at most *optimal_group_size* items in each group.""" if len(items) / optimal_group_size <= max_groups: return chunked(items, optimal_group_size) else: return distribute(max_groups, items)
def divide_grid_in_blocs(self, grid): """ Permet de diviser une grille en neuf blocs différents :returns: list """ return list( flatten( map(lambda row: map(list, distribute(3, chunked(row, 3))), chunked(flatten(grid), 27))))
def download_genomes_parallel(accessions, out_dir=None, threads=1): logger.debug(f"Start downloading genomes using {threads} threads.") list_of_accessions = distribute( threads, accessions) # divide accession list into num of threads futures = [] with ThreadPoolExecutor(max_workers=threads, thread_name_prefix="thread") as executor: for _accessions in list_of_accessions: f = executor.submit(download_genomes_from_assembly, _accessions, out_dir) futures.append(f) [f.result() for f in as_completed(futures)] # wait until all the jobs finish
def reduce_p(function, sequence, n_jobs=multiprocessing.cpu_count(), initial=None): if n_jobs == 1: return reduce(function, sequence, initial) else: sliced_sequence = mit.distribute(n_jobs, sequence) with multiprocessing.Pool(n_jobs) as pool: new_sequence = pool.starmap( reduce, zip(repeat(function), sliced_sequence, repeat(initial))) return reduce(function, new_sequence, initial)
def recombine_into(mating_group: t.List[t.Tuple[GraphIndividual, Record]], brood_size: int) -> t.List[GraphIndividual]: """ Take all genes and distribute them into progeny. For two individuals with N genes and `brood_size=1`, the single offspring will have 2N genes. :param mating_group: A group of individuals selected to give progeny. :param brood_size: A number of offsprings. :return: List of offsprings. """ pool = random_permutation( chain.from_iterable(ind.genes() for ind, _ in mating_group)) chunks = distribute(brood_size, pool) return _init_chunks(mating_group, chunks)
def recombine_genes_uniformly(mating_group: t.List[t.Tuple[Individual, Record]], brood_size: int) -> t.List[Individual]: """ Combines genes of individuals in the `mating_group` in a single pool, and uniformly divides these genes into `brood_size` number of individuals. :param mating_group: A group of individuals selected to give progeny. :param brood_size: A number of offsprings. :return: List of offsprings. """ pool = random_permutation( chain.from_iterable(ind.genes() for ind, _ in mating_group)) chunks = take(brood_size, distribute(len(mating_group), pool)) return _init_chunks(mating_group, chunks)
def print(self, testcases, dryrun, workers, testcases_per_file): """ Multiprocess print testcases to files. Args: testcases (iterable): An iterator of testcases. dryrun (bool): Whether dryrun mode is enabled. workers (int): The number of processes to create. testcases_per_file (int, optional): The maximum number of testcases that can be printing int a single file. """ chunks = distribute(workers, testcases) jobs = [] for i in range(workers): p = Process(target=self._print, args=(chunks[i], i, dryrun, testcases_per_file)) jobs.append(p) p.start() [p.join() for p in jobs]
def test_exchange_fraction(random_genes): g1, g2, g3, g4 = (EdgeGene(1, 2, 'A', 'B', 1, 1), EdgeGene(1, 2, 'A', 'C', 1, 1), EdgeGene(3, 4, 'A', 'B', 1, 1), EdgeGene(3, 4, 'A', 'C', 1, 1)) ind1 = GraphIndividual([g1]) res = exchange_fraction([(ind1, Record(0, 0))], 1, 1.0) assert len(res) == 1 ind1_ = res[0] assert set(ind1_.genes()) == {g1} ind2 = GraphIndividual([g1]) ind1_, ind2_ = exchange_fraction([(ind1, Record(0, 0)), (ind2, Record(0, 0))], 2, 1.0) assert len(ind1_) == len(ind2_) == 1 assert set(ind1_.genes()) == set(ind2_.genes()) == {g1} ind2 = GraphIndividual([g2]) ind1_, ind2_ = exchange_fraction([(ind1, Record(0, 0)), (ind2, Record(0, 0))], 2, 0.9) assert set(ind1_.genes()) == {g1} and set(ind2_.genes()) == {g2} ind1 = GraphIndividual([g1, g2]) ind2 = GraphIndividual([g3, g4]) res = exchange_fraction([(ind1, Record(0, 0)), (ind2, Record(0, 0))], 2, 0.51) assert len(res) == 2 ind1_, ind2_ = res assert len(ind1_) == len(ind2_) == 2 genes1, genes2 = distribute(2, random_graph_genes) ind1, ind2 = GraphIndividual(genes1), GraphIndividual(genes2) res = exchange_fraction([(ind1, Record(0, 0)), (ind2, Record(0, 0))], 1, 0.5) assert len(res) == 1 ind = res[0] old_genes, new_genes = map(list, partition(lambda g: g in genes1, ind.genes())) assert len(old_genes) >= len(new_genes) assert set(new_genes).issubset(genes2)
def run(self, dryrun=False, workers=1, testcases_per_file=1000): """ Run the generator: generate all testcases and print them to files. Args: dryrun (bool, optional): No files are actually written when dryrun is enabled. Defaults to False. workers (int, optional): The number of processes to use. Defaults to 1. testcases_per_file (int, optional): The maximum number of testcases that can be printing int a single file. Defaults to 1000. """ ok = isinstance(dryrun, bool) ok &= isinstance(workers, int) ok &= isinstance(testcases_per_file, int) if not ok: message = 'Bad input types.' self.logger.error(f'TypeError: {message}') raise TypeError(message) ok &= workers > 0 ok &= testcases_per_file > 0 if not ok: message = 'Bad input values.' self.logger.error(f'ValueError: {message}') raise ValueError(message) self.logger.info(f'Generating {self.testcases_length} testcases...') # Make partitions self.logger.debug( f'STEP 1. Finding all possible ways in which {len(self.nodes)} ' f'nodes can be partitioned into {self.number_of_partitions} ' 'partitions...') partitions = self.make_partitions() self.logger.debug( f'{self.number_of_nodes} nodes can be partitioned into ' f'{self.number_of_partitions} partitions in ' f'{self.S(len(self.nodes), self.number_of_partitions)} ways.') # Combine partitions with leaders self.logger.debug( 'STEP 2. Finding all possible ways in which ' f'{self.S(len(self.nodes), self.number_of_partitions)} partitions ' f'can be combined with {len(self.target_nodes)} leaders...') scenarios = self.combine_partitions_with_leaders(partitions) self.logger.debug( f'{self.S(len(self.nodes), self.number_of_partitions)} partitions ' f'can be combined with {len(self.target_nodes)} leaders in ' f'{int(self.testcases_length**(1/self.number_of_rounds))} ' f'possible ways.') # Combine the parition-leader scenarios from above with rounds self.logger.debug( 'STEP 3. Finding all possible ways in which ' f'{int(self.testcases_length**(1/self.number_of_rounds))} ' 'parition-leader scenarios combinations can be combined with ' f'{self.number_of_rounds} rounds...') testcases = self.combine_scenarios_with_rounds(scenarios) self.logger.debug( f'{int(self.testcases_length**(1/self.number_of_rounds))} ' 'parition-leader scenarios can be combined with ' f'{self.number_of_rounds} rounds in {self.testcases_length} ' 'possible ways.') # Print the resulting testcases to files self.logger.debug( f'Printing all testcases to file using {workers} processes...') testcases = distribute(self.number_of_machines, testcases) context_manager = TemporaryDirectory() if dryrun else nullcontext() with context_manager as directory: self.folder_path = self.folder_path if not dryrun else directory self.print(testcases[self.machine_index - 1], dryrun, workers, testcases_per_file) self.logger.info(f'Finished.')
def deal(deck, nPlayers): return [(list(p)) for p in mit.distribute(nPlayers, deck)]
def update_pauli_string(circuit: Circuit, pauli_string: {Circuit, dict[str, float]}, phase: float = 1, parallel: {bool, int} = False, return_info: bool = False, use_mpi: bool = None, compress: int = 4, simplify: bool = True, remove_id_gates: bool = True, float_type: any = 'float32', verbose: bool = False, **kwargs) -> defaultdict: """ Evolve density matrix accordingly to `circuit` using `pauli_string` as initial product state. The evolved density matrix will be represented as a set of different Pauli strings, each of them with a different phase, such that their sum corresponds to the evolved density matrix. The number of branches depends on the number of non-Clifford gates in `circuit`. Parameters ---------- circuit: Circuit Circuit to use to evolve `pauli_string`. pauli_string: {Circuit, dict[str, float]} Pauli string to be evolved. `pauli_string` must be a `Circuit` composed of single qubit Pauli `Gate`s (that is, either `Gate('I')`, `Gate('X')`, `Gate('Y')` or `Gate('Z')`), each one acting on every qubit of `circuit`. If a dictionary is provided, every key of `pauli_string` must be a valid Pauli string. The size of each Pauli string must be equal to the number of qubits in `circuit`. Values in `pauli_string` will be used as inital phase for the given string. phase: float, optional Initial phase for `pauli_string`. atol: float, optional Discard all Pauli strings that have an absolute amplitude smaller than `atol`. parallel: int, optional Parallelize simulation (where possible). If `True`, the number of available cpus is used. Otherwise, a `parallel` number of threads is used. return_info: bool Return extra information collected during the evolution. use_mpi: bool, optional Use `MPI` if available. Unless `use_mpi=False`, `MPI` will be used if detected (for instance, if `mpiexec` is used to called HybridQ). If `use_mpi=True`, force the use of `MPI` (in case `MPI` is not automatically detected). compress: int, optional Compress `Circuit` using `utils.compress` prior the simulation. simplify: bool, optional Simplify `Circuit` using `utils.simplify` prior the simulation. remove_id_gates: bool, optional Remove `ID` gates prior the simulation. float_type: any, optional Float type to use for the simulation. verbose: bool, optional Verbose output. Returns ------- dict[str, float] [, dict[any, any]] If `return_info=False`, `update_pauli_string` returns a `dict` of Pauli strings and the corresponding amplitude. The full density matrix can be reconstructed by resumming over all the Pauli string, weighted with the corresponding amplitude. If `return_info=True`, information gathered during the simulation are also returned. Other Parameters ---------------- eps: float, optional (default: auto) Do not branch if the branch weight for the given non-Clifford operation is smaller than `eps`. `atol=1e-7` if `float_type=float32`, otherwise `atol=1e-8` if `float_type=float64`. atol: float, optional (default: auto) Remove elements from final state if such element as an absolute amplitude smaller than `atol`. `atol=1e-8` if `float_type=float32`, otherwise `atol=1e-12` if `float_type=float64`. branch_atol: float, optional Stop branching if the branch absolute amplitude is smaller than `branch_atol`. If not specified, it will be equal to `atol`. max_breadth_first_branches: int (default: auto) Max number of branches to collect using breadth first search. The number of branches collect during the breadth first phase will be split among the different threads (or nodes if using `MPI`). n_chunks: int (default: auto) Number of chunks to divide the branches obtained during the breadth first phase. The default value is twelve times the number of threads. max_virtual_memory: float (default: 80) Max virtual memory (%) that can be using during the simulation. If the used virtual memory is above `max_virtual_memory`, `update_pauli_string` will raise an error. sleep_time: float (default: 0.1) Completition of parallel processes is checked every `sleep_time` seconds. Example ------- >>> from hybridq.circuit import utils >>> import numpy as np >>> >>> # Define circuit >>> circuit = Circuit( >>> [Gate('X', qubits=[0])**1.2, >>> Gate('ISWAP', qubits=[0, 1])**2.3]) >>> >>> # Define Pauli string >>> pauli_string = Circuit([Gate('Z', qubits=[1])]) >>> >>> # Get density matrix decomposed in Pauli strings >>> dm = clifford.update_pauli_string(circuit=circuit, >>> pauli_string=pauli_string, >>> float_type='float64') >>> >>> dm defaultdict(<function hybridq.circuit.simulation.clifford.update_pauli_string.<locals>._db_init.<locals>.<lambda>()>, {'IZ': 0.7938926261462365, 'YI': -0.12114687473997318, 'ZI': -0.166744368113685, 'ZX': 0.2377641290737882, 'YX': -0.3272542485937367, 'XY': -0.40450849718747345}) >>> # Reconstruct density matrix >>> U = sum(phase * np.kron(Gate(g1).matrix(), >>> Gate(g2).matrix()) for (g1, g2), phase in dm.items()) >>> >>> U array([[ 0.62714826+0.j , 0.23776413+0.j , 0. +0.12114687j, 0. +0.73176275j], [ 0.23776413+0.j , -0.96063699+0.j , 0. -0.07725425j, 0. +0.12114687j], [ 0. -0.12114687j, 0. +0.07725425j, 0.96063699+0.j , -0.23776413+0.j ], [ 0. -0.73176275j, 0. -0.12114687j, -0.23776413+0.j , -0.62714826+0.j ]]) >>> np.allclose(utils.matrix(circuit + pauli_string + circuit.inv()), >>> U, >>> atol=1e-8) True >>> U[0b11, 0b11] (-0.6271482580325515+0j) """ # ==== Set default parameters ==== # If use_mpi==False, force the non-use of MPI if use_mpi is None and _detect_mpi: # Warn that MPI is used because detected warn("MPI has been detected. Using MPI.") # Set MPI to true use_mpi = True # If parallel==True, use number of cpus if type(parallel) is bool: parallel = cpu_count() if parallel else 1 else: parallel = int(parallel) if parallel <= 0: warn("'parallel' must be a positive integer. Setting parallel=1") parallel = 1 # utils.globalize may not work properly on MacOSX systems .. for now, let's # disable parallelization for MacOSX if parallel > 1: from platform import system from warnings import warn if system() == 'Darwin': warn( "'utils.globalize' may not work on MacOSX. Disabling parallelization." ) parallel = 1 # Fix atol if 'atol' in kwargs: atol = kwargs['atol'] del (kwargs['atol']) else: float_type = np.dtype(float_type) if float_type == np.float64: atol = 1e-12 elif float_type == np.float32: atol = 1e-8 else: raise ValueError(f'Unsupported array dtype: {float_type}') # Fix branch_atol if 'branch_atol' in kwargs: branch_atol = kwargs['branch_atol'] del (kwargs['branch_atol']) else: branch_atol = atol # Fix eps if 'eps' in kwargs: eps = kwargs['eps'] del (kwargs['eps']) else: float_type = np.dtype(float_type) if float_type == np.float64: eps = 1e-8 elif float_type == np.float32: eps = 1e-7 else: raise ValueError(f'Unsupported array dtype: {float_type}') # Set default db initialization def _db_init(): return defaultdict(int) # Set default transform def _transform(ps): # Join bitstring return ''.join({_X: 'X', _Y: 'Y', _Z: 'Z', _I: 'I'}[op] for op in ps) # Set default collect def _collect(db, ps, ph): # Update final paulis db[ps] += ph # Remove elements close to zero if abs(db[ps]) < atol: del (db[ps]) # Set default merge def _merge(db, db_new, use_tuple=False): # Update final paulis for ps, ph in db_new if use_tuple else db_new.items(): # Collect results kwargs['collect'](db, ps, ph) kwargs.setdefault('max_breadth_first_branches', min(4 * 12 * parallel, 2**14)) kwargs.setdefault('n_chunks', 12 * parallel) kwargs.setdefault('max_virtual_memory', 80) kwargs.setdefault('sleep_time', 0.1) kwargs.setdefault('collect', _collect) kwargs.setdefault('transform', _transform) kwargs.setdefault('merge', _merge) kwargs.setdefault('db_init', _db_init) # Get MPI info if use_mpi: from mpi4py import MPI _mpi_comm = MPI.COMM_WORLD _mpi_size = _mpi_comm.Get_size() _mpi_rank = _mpi_comm.Get_rank() kwargs.setdefault('max_breadth_first_branches_mpi', min(_mpi_size * 2**9, 2**14)) kwargs.setdefault('mpi_chunk_max_size', 2**20) kwargs.setdefault('mpi_merge', True) # Get complex_type from float_type complex_type = (np.array([1], dtype=float_type) + 1j * np.array([1], dtype=float_type)).dtype # Local verbose _verbose = verbose and (not use_mpi or _mpi_rank == 0) # =========== CHECKS ============= if type(pauli_string) == Circuit: from collections import Counter # Initialize error message _err_msg = "'pauli_string' must contain only I, X, Y and Z gates acting on different qubits." # Check qubits match with circuit if any(g.n_qubits != 1 or not g.qubits for g in pauli_string) or set( pauli_string.all_qubits()).difference( circuit.all_qubits()) or set( Counter(gate.qubits[0] for gate in pauli_string).values()).difference( [1]): raise ValueError(_err_msg) # Get ideal paulis _ig = list(map(lambda n: Gate(n).matrix(), 'IXYZ')) # Get the correct pauli def _get_pauli(gate): # Get matrix U = gate.matrix() # Get right pauli p = next( (p for x, p in enumerate('IXYZ') if np.allclose(_ig[x], U)), None) # If not found, raise error if not p: raise ValueError(_err_msg) # Otherwise, return pauli return Gate(p, qubits=gate.qubits) # Reconstruct paulis pauli_string = Circuit(map(_get_pauli, pauli_string)) else: # Check that all strings only have I,X,Y,Z tokens _n_qubits = len(circuit.all_qubits()) if any( set(p).difference('IXYZ') or len(p) != _n_qubits for p in pauli_string): raise ValueError( f"'pauli_string' must contain only I, X, Y and Z gates acting on different qubits." ) # ================================ # Start pre-processing time _prep_time = time() # Get qubits _qubits = circuit.all_qubits() # Remove ID gates if remove_id_gates: circuit = Circuit(gate for gate in circuit if gate.name != 'I') # Simplify circuit if simplify: # Get qubits to pin if type(pauli_string) == Circuit: # Pinned qubits _pinned_qubits = pauli_string.all_qubits() else: # Find qubits to pin _pinned_qubits = set.union( *({q for q, g in zip(_qubits, p) if g != 'I'} for p in pauli_string)) # Simplify circuit = utils.simplify(circuit, remove_id_gates=remove_id_gates, verbose=_verbose) circuit = utils.popright(utils.simplify(circuit), pinned_qubits=set(_pinned_qubits).intersection( circuit.all_qubits()), verbose=_verbose) # Compress circuit circuit = Circuit( utils.to_matrix_gate(c, complex_type=complex_type) for c in tqdm(utils.compress(circuit, max_n_qubits=compress), disable=not _verbose, desc=f"Compress ({int(compress)})")) # Pad missing qubits circuit += Circuit( Gate('MATRIX', [q], U=np.eye(2)) for q in set(_qubits).difference(circuit.all_qubits())) # Get qubits map qubits_map = kwargs['qubits_map'] if 'qubits_map' in kwargs else { q: x for x, q in enumerate(circuit.all_qubits()) } # Pre-process circuit _LS_cache = {} _P_cache = {} circuit = [ g for gate in tqdm(reversed(circuit), total=len(circuit), disable=not _verbose, desc='Pre-processing') for g in _process_gate(gate, LS_cache=_LS_cache, P_cache=_P_cache) ] _LS_cache.clear() _P_cache.clear() del (_LS_cache) del (_P_cache) # Get maximum number of qubits and parameters _max_n_qubits = max(max(len(gate[1]) for gate in circuit), 2) _max_n_params = max(len(gate[2]) for gate in circuit) # Get qubits qubits = np.array([ np.pad([qubits_map[q] for q in gate[1]], (0, _max_n_qubits - len(gate[1]))) for gate in circuit ], dtype='int32') # Get parameters params = np.round( np.array([ np.pad(gate[2], (0, _max_n_params - len(gate[2]))) for gate in circuit ], dtype=float_type), -int(np.floor(np.log10(atol))) if atol < 1 else 0) # Remove -0 params[np.abs(params) == 0] = 0 # Quick check assert (all('_' + gate[0] in globals() for gate in circuit)) # Get gates gates = np.array([globals()['_' + gate[0]] for gate in circuit], dtype='int') # Compute expected number of paths _log2_n_expected_branches = 0 for _idx in np.where(np.isin(gates, _MATRIX_SET))[0]: _nq = (gates[_idx] // _gate_mul) + 1 _p = params[_idx][:4**(2 * _nq)] _log2_n_expected_branches += np.sum( np.log2( np.sum(np.abs(np.reshape(_p, (4**_nq, 4**_nq))) > eps, axis=1))) / 4**_nq # Check assert (len(gates) == len(qubits) and len(gates) == len(params)) # Initialize branches if type(pauli_string) == Circuit: # Convert Pauli string _pauli_string = np.array([_I] * len(qubits_map), dtype='int') for gate in pauli_string: if gate.name != 'I': _pauli_string[qubits_map[gate.qubits[0]]] = { 'X': _X, 'Y': _Y, 'Z': _Z }[gate.name] # Initialize branches branches = [(_pauli_string, phase, 0)] else: # Initialize branches branches = [(np.array([{ 'I': _I, 'X': _X, 'Y': _Y, 'Z': _Z }[g] for g in p], dtype='int'), phase, 0) for p, phase in pauli_string.items() if abs(phase) > atol] # Initialize final Pauli strings db = kwargs['db_init']() # Define update function _update = partial_func(_update_pauli_string, gates, qubits, params, eps=eps, atol=branch_atol) # End pre-processing time _prep_time = time() - _prep_time # Initialize infos _info_init = lambda: { 'n_explored_branches': 0, 'largest_n_branches_in_memory': 0, 'peak_virtual_memory (GB)': virtual_memory().used / 2**30, 'average_virtual_memory (GB)': (virtual_memory().used / 2**30, 1), 'n_threads': parallel, 'n_cpus': cpu_count(), 'eps': eps, 'atol': atol, 'branch_atol': branch_atol, 'float_type': str(float_type), 'log2_n_expected_branches': _log2_n_expected_branches } infos = _info_init() infos['memory_baseline (GB)'] = virtual_memory().used / 2**30 if not use_mpi or _mpi_rank == 0: infos['n_explored_branches'] = 1 infos['largest_n_branches_in_memory'] = 1 # Start clock _init_time = time() # Scatter first batch of branches to different MPI nodes if use_mpi and _mpi_size > 1: if _mpi_rank == 0: # Explore branches (breadth-first search) branches = _breadth_first_search( _update, db, branches, max_n_branches=kwargs['max_breadth_first_branches_mpi'], infos=infos, verbose=verbose, mpi_rank=_mpi_rank, **kwargs) # Distribute branches branches = _mpi_comm.scatter( [list(x) for x in distribute(_mpi_size, branches)], root=0) # Explore branches (breadth-first search) branches = _breadth_first_search( _update, db, branches, max_n_branches=kwargs['max_breadth_first_branches'], infos=infos, verbose=verbose if not use_mpi or _mpi_rank == 0 else False, **kwargs) # If there are remaining branches, use depth-first search if branches: _depth_first_search(_update, db, branches, parallel=parallel, infos=infos, info_init=_info_init, verbose=verbose, mpi_rank=_mpi_rank if use_mpi else 0, mpi_size=_mpi_size if use_mpi else 1, **kwargs) # Update infos infos['average_virtual_memory (GB)'] = infos['average_virtual_memory (GB)'][ 0] / infos['average_virtual_memory (GB)'][1] - infos[ 'memory_baseline (GB)'] infos['peak_virtual_memory (GB)'] -= infos['memory_baseline (GB)'] # Update branching time infos['branching_time (s)'] = time() - _init_time # Collect results if use_mpi and _mpi_size > 1 and kwargs['mpi_merge']: for _k in infos: infos[_k] = [infos[_k]] # Initialize pbar if _mpi_rank == 0: pbar = tqdm(total=int(np.ceil(np.log2(_mpi_size))), disable=not verbose, desc='Collect results') # Initialize tag and size _tag = 0 _size = _mpi_size while _size > 1: # Update progressbar if _mpi_rank == 0: pbar.set_description( f'Collect results (Mem={virtual_memory().percent}%)') # Get shift _shift = (_size // 2) + (_size % 2) if _mpi_rank < (_size // 2): # Get infos _infos = _mpi_comm.recv(source=_mpi_rank + _shift, tag=_tag) # Update infos for _k in infos: infos[_k].extend(_infos[_k]) # Get number of chunks _n_chunks = _mpi_comm.recv(source=_mpi_rank + _shift, tag=_tag + 1) if _n_chunks > 1: # Initialize _process with tqdm(range(_n_chunks), desc='Get db', leave=False, disable=_mpi_rank != 0) as pbar: for _ in pbar: # Receive db _db = _mpi_comm.recv(source=_mpi_rank + _shift, tag=_tag + 2) # Merge datasets kwargs['merge'](db, _db, use_tuple=True) # Update description pbar.set_description( f'Get db (Mem={virtual_memory().percent}%)') # Clear dataset _db.clear() else: # Receive db _db = _mpi_comm.recv(source=_mpi_rank + _shift, tag=_tag + 2) # Merge datasets kwargs['merge'](db, _db) # Clear dataset _db.clear() elif _shift <= _mpi_rank < _size: # Remove default_factory because pickle is picky regarding local objects db.default_factory = None # Send infos _mpi_comm.send(infos, dest=_mpi_rank - _shift, tag=_tag) # Compute chunks _n_chunks = kwargs['mpi_chunk_max_size'] _n_chunks = (len(db) // _n_chunks) + ( (len(db) % _n_chunks) != 0) # Send number of chunks _mpi_comm.send(_n_chunks, dest=_mpi_rank - _shift, tag=_tag + 1) if _n_chunks > 1: # Split db in chunks for _db in chunked(db.items(), kwargs['mpi_chunk_max_size']): _mpi_comm.send(_db, dest=_mpi_rank - _shift, tag=_tag + 2) else: # Send db _mpi_comm.send(db, dest=_mpi_rank - _shift, tag=_tag + 2) # Reset db and infos db.clear() infos.clear() # update size _tag += 3 _size = _shift _mpi_comm.barrier() # Update progressbar if _mpi_rank == 0: pbar.set_description( f'Collect results (Mem={virtual_memory().percent}%)') pbar.update() # Update runtime if not use_mpi or _mpi_rank == 0 or not kwargs['mpi_merge']: infos['runtime (s)'] = time() - _init_time infos['pre-processing (s)'] = _prep_time # Check that all the others dbs/infos (excluding rank==0) has been cleared up if use_mpi and _mpi_rank > 0 and kwargs['mpi_merge']: assert (not len(db) and not len(infos)) if return_info: return db, infos else: return db
def _depth_first_search(_update, db, branches, parallel, infos, info_init, verbose, mpi_rank, mpi_size, **kwargs): # Define parallel core def _parallel_core(branches, db=None): # Initialize db if db is None: db = kwargs['db_init']() # Convert to list branches = list(branches) # Initialize infos infos = info_init() # Explore all branches while branches: # Get new branches (_new_ps, _new_ph), _new_branches = _update(*branches.pop()) # Collect results kwargs['collect'](db, kwargs['transform'](_new_ps), _new_ph) # Update branches branches.extend(_new_branches) # Update infos infos['largest_n_branches_in_memory'] = max( len(branches), infos['largest_n_branches_in_memory']) # Update infos infos['n_explored_branches'] += 1 return db, infos # If no parallelization is requires, explore branchces one by one if parallel == 1: from more_itertools import ichunked # Get number of chunks chunk_size = max(1, len(branches) // 100) for _bs in tqdm(ichunked(branches, chunk_size), total=len(branches) // chunk_size, desc=f'Mem={virtual_memory().percent}%', disable=not verbose): # Update database and infos db, _infos = _parallel_core(_bs, db) # Update infos infos['n_explored_branches'] += _infos['n_explored_branches'] infos['largest_n_branches_in_memory'] = max( _infos['largest_n_branches_in_memory'], infos['largest_n_branches_in_memory']) # Otherwise, distribute workload among different cores else: with globalize(_parallel_core) as _parallel_core, Pool( parallel) as pool: # Apply async _fps = [ pool.apply_async(_parallel_core, (_branches,)) for _branches in distribute(kwargs['n_chunks'], branches) ] _status = [False] * len(_fps) with tqdm(total=len(_fps), desc=f'Mem={virtual_memory().percent}%', disable=not verbose) as pbar: _pending = len(_fps) while _pending: # Wait sleep(kwargs['sleep_time']) # Activate/disactivate if verbose: pbar.disable = int(time()) % mpi_size != mpi_rank # Get virtual memory _vm = virtual_memory() _pending = 0 for _x, (_p, _s) in enumerate(zip(_fps, _status)): if not _p.ready(): _pending += 1 elif not _s: # Collect data _new_db, _infos = _p.get() # Merge datasets kwargs['merge'](db, _new_db) # Clear dataset _new_db.clear() # Update infos infos['n_explored_branches'] += _infos[ 'n_explored_branches'] infos['largest_n_branches_in_memory'] = max( _infos['largest_n_branches_in_memory'], infos['largest_n_branches_in_memory']) # Set status _status[_x] = True # Update pbar if verbose: pbar.set_description( (f'[{mpi_rank}] ' if mpi_size > 1 else '') + \ f'Mem={_vm.percent}%, ' + \ f'NThreads={infos["n_threads"]}, ' + \ f'NCPUs={infos["n_cpus"]}, ' + \ f'LoadAvg={getloadavg()[0]/infos["n_cpus"]*100:1.2f}%, ' + \ f'NBranches={infos["n_explored_branches"]}' ) pbar.n = len(_fps) - _pending pbar.refresh() # Update infos infos['average_virtual_memory (GB)'] = ( infos['average_virtual_memory (GB)'][0] + _vm.used / 2**30, infos['average_virtual_memory (GB)'][1] + 1) infos['peak_virtual_memory (GB)'] = max( infos['peak_virtual_memory (GB)'], _vm.used / 2**30) # If memory above threshold, raise error if _vm.percent > kwargs['max_virtual_memory']: raise MemoryError( f'Memory above threshold: {_vm.percent}% > {kwargs["max_virtual_memory"]}%' ) # Last refresh if verbose: pbar.refresh() # Check all chunks have been explored assert (np.alltrue(_status))
import random import more_itertools as mit import operator # Build a Deck suits = "CDHS" ranks = list(range(2, 11)) + list("JQKA") for i in range(0, len(ranks)): ranks[i] = str(ranks[i]) DeckCard = [j + i for j in suits for i in ranks] # Shuffle and Distribute players = 4 random.shuffle(DeckCard) hands = [list(hand) for hand in list(mit.distribute(players, DeckCard))] #return all 4 hands to each player print("player1:", hands[0]) print("player2:", hands[1]) print("player3:", hands[2]) print("AgentHand:", hands[3]) playcard1 = 'SA' playcard2 = 'SK' playcard3 = 'SQ' playcard4 = 'S10' trick = [playcard1, playcard2, playcard3, playcard4] print(len(trick)) if len(trick) == 4: leadSuit = trick[0][1] print(leadSuit) cards_value = {'2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9, '10': 10, 'J': 11, 'Q': 12, 'K': 13, 'A': 14 }
from AnalyticalReasoning import Variable import more_itertools as m_it n = 5 group = numpy.empty(n, dtype=object) A = Variable("A") B = Variable("B") C = Variable("C") D = Variable("D") E = Variable("E") F = Variable("F") G = Variable("G") H = Variable("H") I = Variable("I") group[0] = I group[1] = H group[2] = G group[3] = F group[4] = E print(len(group)) children = m_it.distribute(3, [1, 2, 3, 4, 5, 6, 7]) distinct_lists = [] for perm in it.permutations([1, 2, 3, 4, 5, 6]): lists = [set(s) for s in m_it.split_into(perm, [1, 2, 3])] #print(lists) if (lists not in distinct_lists): distinct_lists.append(lists) print(lists) #print(distinct_lists)