def create_process_pool(dic_num=-1):
if dic_num > 0:
set_cpu_num(dic_num)
cnum = get_cpu_num()
print_log_info(
['creating multiprocessing pool. process number is ', cnum, '.'])
return multiprocessing.Pool(processes=cnum)
def check_threadpool_recreate(threadpool):
# 重建进程池
if init_vars.need_recreate:
terminate_threadpool(threadpool)
threadpool = create_thread_pool()
print_log_info('recreating multithreading pool success.')
init_vars.need_recreate = False
return threadpool
def init_ws_server():
cmd = ['python', 'ws_server.py']
command = ' '.join(cmd)
print_log_info(['running ws server...'])
try:
subprocess.Popen(command, shell=False, cwd=script_path)
except Exception as e:
print(e)
def init_wd_server():
if check_system() == 0:
cmd = ['python3', 'wd_server.py']
command = ' '.join(cmd)
shell = True
else:
cmd = ['python', 'wd_server.py']
command = ' '.join(cmd)
shell = False
print_log_info(['running watch dog server...'])
try:
subprocess.Popen(command, shell=shell, cwd=script_path)
except Exception as e:
print(e)
def rewrite_cache(tmdict_root_path):
global init_vars
t1 = time.perf_counter()
sound_list.clear()
get_sound_list()
init_vars.mdict_odict.clear()
init_vars.mdict_odict, init_vars.zim_list = get_mdict_dict(tmdict_root_path)
init_vars.mdict_odict, init_vars.indicator = sort_mdict_list(init_vars.mdict_odict)
t2 = time.perf_counter()
print_log_info('initializing mdict_list', 0, t1, t2)
write_cache()
write_dir_change()
init_vars.mtime = os.path.getmtime(pickle_file_path)
print_log_info('creating cache file', 0, t2, time.perf_counter())
def extract_index_from_zim(root, zim):
for url, index in zim.index_list:
url = url.replace('/', '_')
idx_name = zim.get_fname() + '_' + url + '.idx'
idx_path = os.path.join(root, idx_name)
if not os.path.exists(idx_path):
t1 = time.perf_counter()
zim_file = open(zim.get_fpath(), 'rb')
idx_data = zim._get_article_by_index(zim_file, index)[0]
zim_file.close()
if idx_data is None:
continue
with open(idx_path, 'wb') as f:
f.write(idx_data)
t2 = time.perf_counter()
print_log_info(['index extracting', idx_name], start=t1, end=t2)
def init_mdict_list():
global init_vars, mdict_root_path
t1 = time.perf_counter()
rename_history()
if check_dir_change():
rewrite_cache(mdict_root_path)
else:
get_sound_list()
load_cache(mdict_root_path)
init_zim_list()
print_log_info('reading from cache file', 0, t1, time.perf_counter())
print_log_info(['media root path:', mdict_root_path])
print_log_info(['audio root path:', audio_path])
print_log_info(['dictionary counts', len(init_vars.mdict_odict)])
return init_vars
def terminate_threadpool(threadpool):
print_log_info('terminating multithreading pool.')
threadpool.terminate()
def create_thread_pool():
cnum = get_cpu_num()
print_log_info(
['creating multithreading pool. thread number is ', cnum, '.'])
return ThreadPool(cnum)
def extract_index(idx_list):
t1 = time.perf_counter()
for root, zim in idx_list:
extract_index_from_zim(root, zim)
t2 = time.perf_counter()
print_log_info('all indexes extracting completed.', start=t1, end=t2)
def get_mdict_dict(tmdict_root_path):
g_id = 0
m_dict = collections.OrderedDict()
idx_list = []
zim_list = []
# 词典结尾是mdx或MDX
# ntfs下文件名不区分大小写，但ext4下区分大小写
for root, dirs, files in os.walk(tmdict_root_path):
for file in files:
if file.lower().endswith('.mdx'):
f_name = file[:file.rfind('.')]
mdx_path = os.path.join(root, file)
mdd_list = []
for f in files:
if f.lower().endswith('.mdd') and f.startswith(f_name):
f2_name = f[:f.rfind('.')]
if f2_name != f_name:
if f2_name + '.mdx' in files:
continue
mdd_path = os.path.join(root, f)
try:
mdd_list.append(MDD(mdd_path))
except Exception as e:
print_log_info([f_name, 'mdd loading failed', e])
if mdx_path.find('.part') != -1:
for item in m_dict.values():
x = item.mdx
g = item.g_id
if x.get_fpath().find('.part'):
x_fname = os.path.basename(x.get_fpath())
if x_fname[:x_fname.rfind(
'.part')] == f_name[:f_name.rfind('.part'
)]:
g_id = g
break
icon = 'none'
for f in files:
if f.startswith(f_name):
mime_type = guess_mime(f)
if mime_type is not None and mime_type.startswith(
'image'):
if f[:f.rfind('.')] == f_name:
icon = f.split('.')[-1]
break
try:
mdx = MDX(mdx_path)
m_dict.update({
f_name:
MdictItem(mdx, tuple(mdd_list), g_id, icon,
mdx.get_len())
})
except Exception as e:
print_log_info([f_name, 'mdx loading failed', e])
g_id += 1
elif file.lower().endswith('.zim'):
f_name = file[:file.rfind('.')]
zim_path = os.path.join(root, file)
zim = ZIMFile(zim_path, encoding='utf-8')
m_dict.update(
{f_name: MdictItem(zim, [], -1, 'none', len(zim))})
zim_list.append(zim)
idx_list.append((root, zim))
if idx_list and check_xapian():
# 抽取zim内置索引
tdx = threading.Thread(target=extract_index, args=(idx_list, ))
tdx.start()
return m_dict, zim_list
def check_dir_change():
old_dir = read_change()
if old_dir is None:
print_log_info('dat cache file not exists.')
return True
if old_dir['root_dir'] != mdict_root_path:
print_log_info('mdict_root_path has changed.')
return True
if not os.path.exists(mdict_root_path):
print_log_info('mdict_root_path not exists.')
return True
files_total_num = 1
for root, dirs, files in os.walk(mdict_root_path):
for file in files:
if file.lower().endswith('.mdx') or file.lower().endswith('.mdd') or file.lower().endswith('.zim'):
mdict_path = os.path.join(root, file)
mtime = os.path.getmtime(mdict_path)
files_total_num += 1
if mdict_path not in old_dir.keys():
print_log_info('dir change founded.')
return True
if old_dir[mdict_path] < mtime:
print_log_info('dir change founded.')
return True
if files_total_num != len(old_dir):
print_log_info('dir change founded.')
return True
print_log_info('no dir change founded.')
return False
def terminate_pool(pool):
print_log_info('terminating multiprocessing pool.')
if pool is not None:
pool.terminate()
