def get_keys_multiproc( cls, lookup, loc_df, success_only=False, num_cores=-1, num_partitions=-1 ): """ Used for CPU bound lookup operations, Depends on a method `process_locations_multiproc(dataframe)` where single_row is a pandas series from a location Pandas DataFrame and returns a list of dicts holding the lookup results for that single row """ pool_count = num_cores if num_cores > 0 else cpu_count() part_count = num_partitions if num_partitions > 0 else min(pool_count * 2, len(loc_df)) locations = np.array_split(loc_df, part_count) pool = Pool(pool_count) results = pool.map(lookup.process_locations_multiproc, locations) lookup_results = sum([r for r in results if r], []) pool.terminate() return lookup_results
def verify_cipher_mix(cipher_mix, teller=_teller, nr_parallel=0): try: p = cipher_mix['modulus'] g = cipher_mix['generator'] q = cipher_mix['order'] y = cipher_mix['public'] original_ciphers = cipher_mix['original_ciphers'] mixed_ciphers = cipher_mix['mixed_ciphers'] challenge = cipher_mix['challenge'] cipher_collections = cipher_mix['cipher_collections'] offset_collections = cipher_mix['offset_collections'] random_collections = cipher_mix['random_collections'] except KeyError as e: m = "Invalid cipher mix format" raise ZeusError(m, e) if compute_mix_challenge(cipher_mix) != challenge: m = "Invalid challenge" raise ZeusError(m) nr_ciphers = len(original_ciphers) nr_rounds = len(cipher_collections) teller.task('Verifying mixing of %d ciphers for %d rounds' % (nr_ciphers, nr_rounds)) if (len(offset_collections) != nr_rounds or len(random_collections) != nr_rounds): m = "Invalid cipher mix format: collections not of the same size!" raise ZeusError(m) #if not validate_cryptosystem(p, g, q, teller): # m = "Invalid cryptosystem" # raise AssertionError(m) total = nr_rounds * nr_ciphers with teller.task('Verifying ciphers', total=total): data = [] for i, bit in zip(range(nr_rounds), bit_iterator(int(challenge, 16))): ciphers = cipher_collections[i] randoms = random_collections[i] offsets = offset_collections[i] data.append((p, g, q, y, i, bit, original_ciphers, mixed_ciphers, ciphers, randoms, offsets)) if nr_parallel <= 0: for args in data: verify_mix_round(*args, teller=teller) else: pool = Pool(nr_parallel, Random.atfork) try: for count in pool.imap(_verify_mix_round, data): teller.advance(count) finally: pool.terminate() pool.join() teller.finish('Verifying mixing') return 1