def smp_apply(method, items, progress_callback=None, use_threadpool=False, listener=None, listener_param=tuple(), num_procs_param=None):
"""
@type method: function
@type items: list
@type progress_callback: function
@type use_threadpool: bool
@type listener: function
@type listener_param: tuple
"""
last_update = [time.time()]
results_cnt = [0]
def progress_callback_wrapper(result_func):
"""
progress_callback
@type result_func: object
"""
if progress_callback:
now = time.time()
results_cnt[0] += 1
try:
perc = float(results_cnt[0]) / (float(len(items)) / 100)
except ZeroDivisionError:
perc = 0
if results_cnt[0] == 1 and perc == 100:
pass
else:
if now - last_update[0] > 0.1:
if perc > 100:
perc = 100
progress_callback(perc)
last_update[0] = now
return result_func
num_procs = 8
if num_procs_param:
num_procs = num_procs_param
else:
try:
from multiprocessing import g_cpu_count
num_procs = g_cpu_count()
except Exception, e:
log_json(str(e))
def make_chunklist(fpath):
"""
@type fpath: str
"""
if not os.path.exists(fpath):
raise Exception("make_chunklist: file does not exist")
fstats = os.stat(fpath)
fsize = fstats.st_size
chunksize = (20 * (2 ** 20))
#noinspection PyBroadException
try:
import multiprocessing
numcores = multiprocessing.g_cpu_count()
if (numcores * chunksize) > fsize:
chunksize = int(math.ceil(float(fsize) / numcores))
except:
pass
if chunksize == 0:
chunksize = 1
num_chunks = fsize / chunksize
chunk_remainder = fsize % chunksize
chunklist = [chunksize] * num_chunks
chunklist.append(chunk_remainder)
chunklist_abs = []
val = 0
for i in chunklist:
chunklist_abs.append((val, i))
val += i
if chunk_remainder != 0:
last = chunklist_abs.pop()
chunklist_abs.append((last[0], last[1] + chunk_remainder))
if chunklist_abs[len(chunklist_abs) - 1][1] == 0:
chunklist_abs.pop()
return chunklist_abs
def smp_apply_0(method, items, progress_callback=None):
"""
@type method: function
@type items: list
@type progress_callback: function
"""
pool = Pool(processes=multiprocessing.g_cpu_count())
results_cnt = [0]
last_update = [time.time()]
def progress_callback_wrapper(result_func):
"""
progress_callback
@type result_func: object
"""
if progress_callback:
now = time.time()
results_cnt[0] += 1
try:
perc = float(results_cnt[0]) / (float(len(items)) / 100)
except ZeroDivisionError:
perc = 0
if results_cnt[0] == 1 and perc == 100:
pass
else:
if now - last_update[0] > 0.1:
if perc > 100:
perc = 100
progress_callback(perc)
last_update[0] = now
return result_func
calculation_result = []
for item in items:
base_params_list = []
if isinstance(item, tuple):
for i in item:
base_params_list.append(i)
else:
base_params_list.append(item)
params = tuple(base_params_list)
result = pool.apply_async(method,
params,
callback=progress_callback_wrapper)
calculation_result.append(result)
pool.close()
pool.join()
calculation_result_values = []
for result in calculation_result:
if not result.successful():
result.get()
else:
calculation_result_values.append(result.get())
pool.terminate()
if progress_callback_wrapper:
progress_callback_wrapper(100)
return calculation_result_values
def smp_apply_0(method, items, progress_callback=None):
"""
@type method: function
@type items: list
@type progress_callback: function
"""
pool = Pool(processes=multiprocessing.g_cpu_count())
results_cnt = [0]
last_update = [time.time()]
def progress_callback_wrapper(result_func):
"""
progress_callback
@type result_func: object
"""
if progress_callback:
now = time.time()
results_cnt[0] += 1
try:
perc = float(results_cnt[0]) / (float(len(items)) / 100)
except ZeroDivisionError:
perc = 0
if results_cnt[0] == 1 and perc == 100:
pass
else:
if now - last_update[0] > 0.1:
if perc > 100:
perc = 100
progress_callback(perc)
last_update[0] = now
return result_func
calculation_result = []
for item in items:
base_params_list = []
if isinstance(item, tuple):
for i in item:
base_params_list.append(i)
else:
base_params_list.append(item)
params = tuple(base_params_list)
result = pool.apply_async(method, params, callback=progress_callback_wrapper)
calculation_result.append(result)
pool.close()
pool.join()
calculation_result_values = []
for result in calculation_result:
if not result.successful():
result.get()
else:
calculation_result_values.append(result.get())
pool.terminate()
if progress_callback_wrapper:
progress_callback_wrapper(100)
return calculation_result_values