def get_keys_multiproc(
cls,
lookup,
loc_df,
success_only=False,
num_cores=-1,
num_partitions=-1
):
"""
Used for CPU bound lookup operations, Depends on a method
`process_locations_multiproc(dataframe)`
where single_row is a pandas series from a location Pandas DataFrame
and returns a list of dicts holding the lookup results for that single row
"""
pool_count = num_cores if num_cores > 0 else cpu_count()
part_count = num_partitions if num_partitions > 0 else min(pool_count * 2, len(loc_df))
locations = np.array_split(loc_df, part_count)
pool = Pool(pool_count)
results = pool.map(lookup.process_locations_multiproc, locations)
lookup_results = sum([r for r in results if r], [])
pool.terminate()
return lookup_results
def searchDarkWeb(self, query, include=None, exclude=None):
''' Gets data from search engines specified '''
if include:
final = [a for a in include if a in self.sites]
elif exclude:
final = [a for a in self.sites if a not in exclude]
else:
final = list(self.sites.keys())
self.query = query
pool = Pool(processes=len(final))
data = pool.map(self.search, final)
resultList = [d for dat in data for d in dat]
pool.close()
ind = Indexer()
for i in resultList:
ind.join(i)
return ind.results()
def _scan_match(sample_ui_list, path_list, comp_func, weight_list=None, threshold=0.6, pool_size=12):
"""
:param sample_ui_list: output after process_csv()
:param path_list: relative or absolute path list of csv files
:param comp_func: compare function
:param weight_list: ui weight mask
:param threshold: threshold，超过一定的阈值才会被计算成相同组件
:param pool_size: 并行池大小
:return: best match path name
"""
pool = Pool(processes=pool_size)
arg_list = []
for j in range(len(path_list)):
arg_list.append((j + 1, path_list[j], sample_ui_list, comp_func, weight_list, threshold))
score_list = pool.map(_single_scan_helper, arg_list)
pool.close()
pool.join()
# return sorted path^score^score_distribution_list list
return sorted(score_list, key=lambda k: k[1], reverse=True)
def main():
p = Pool(5)
A = []
for i in range(0, 100):
A.append(i)
print(p.map(f, A))