def load_modules(path='func/minion/modules/', main_class=func_module.FuncModule, blacklist=None, parent_class=None):
python_path = distutils.sysconfig.get_python_lib()
module_file_path = "%s/%s" % (python_path, path)
(mod_path, mod_dir) = os.path.split(os.path.normpath(module_file_path))
mod_dir = "func."+module_file_path[len(python_path+'/func/'):].replace("/",".")
sys.path.insert(0, mod_path)
mods = {}
bad_mods = {}
filenames = module_walker(module_file_path)
# FIXME: this is probably more complicated than it needs to be -akl
for fn in filenames:
# aka, everything after the module_file_path
module_name_part = fn[len(module_file_path):]
dirname, basename = os.path.split(module_name_part)
if basename[:8] == "__init__":
modname = dirname
dirname = ""
elif basename[-3:] == ".py":
modname = basename[:-3]
elif basename[-4:] in [".pyc", ".pyo"]:
modname = basename[:-4]
pathname = modname
if dirname != "":
pathname = "%s/%s" % (dirname, modname)
mod_imp_name = pathname.replace("/", ".")
if mods.has_key(mod_imp_name):
# If we've already imported mod_imp_name, don't import it again
continue
# ignore modules that we've already determined aren't valid modules
if bad_mods.has_key(mod_imp_name):
continue
try:
# Auto-detect and load all FuncModules
blip = __import__("%s%s" % ( mod_dir,mod_imp_name), globals(), locals(), [mod_imp_name])
for obj in dir(blip):
attr = getattr(blip, obj)
if isclass(attr) and issubclass(attr, main_class):
logger.debug("Loading %s module" % attr)
if parent_class:
mods[mod_imp_name] = attr(parent_class)
else:
mods[mod_imp_name] = attr()
except ImportError, e:
# A module that raises an ImportError is (for now) simply not loaded.
errmsg = _("Could not load %s module: %s")
logger.warning(errmsg % (mod_imp_name, e))
bad_mods[mod_imp_name] = True
continue
except:
def run(self):
"""
"""
logger.debug("遍历wav_name,text信息")
wav_suf = "wav"
counter = 0
whole_wavs = set()
wavs_info_map = {}
logger.info(f"按照 【%s】 进行分包!" %
self.custom_classfy if self.custom_classfy else "SPK")
for spk, wav_name, content in self.mklines(): # 遍历说话人id，音频名，音频内容 信息
wav_time = self.wav_time_map.get(wav_name)
if wav_time is None:
logger.warning("未获取到音频时间 [%s]" % wav_name)
continue
wav_info = [ # 填充新平台文本格式
{
"Wav_name":
wav_name,
"Length_time":
wav_time,
"Data": [{
"text": content,
"start_time": 0,
"end_time": wav_time
}],
"Wav_suf":
wav_suf,
}
]
whole_wavs.add(wav_name)
if self.custom_classfy: # 指定分包数的模式
ids = counter
if ids not in wavs_info_map:
wavs_info_map[ids] = [wav_info]
else:
if len(wavs_info_map[ids]) == self.custom_classfy - 1:
counter += 1
wavs_info_map[ids].append(wav_info)
else: # 默认分包模式
if spk not in wavs_info_map:
wavs_info_map[spk] = [wav_info]
else:
wavs_info_map[spk].append(wav_info)
whole_sum = len(whole_wavs)
result = (wavs_info_map, whole_sum)
return result
def load_modules(blacklist=None):
module_file_path = "%s/func/minion/modules/" % distutils.sysconfig.get_python_lib(
)
mod_path = "%s/func/minion/" % distutils.sysconfig.get_python_lib()
sys.path.insert(0, mod_path)
mods = {}
bad_mods = {}
filenames = module_walker(module_file_path)
# FIXME: this is probably more complicated than it needs to be -akl
for fn in filenames:
# aka, everything after the module_file_path
module_name_part = fn[len(module_file_path):]
dirname, basename = os.path.split(module_name_part)
if basename[:8] == "__init__":
modname = dirname
dirname = ""
elif basename[-3:] == ".py":
modname = basename[:-3]
elif basename[-4:] in [".pyc", ".pyo"]:
modname = basename[:-4]
pathname = modname
if dirname != "":
pathname = "%s/%s" % (dirname, modname)
mod_imp_name = pathname.replace("/", ".")
if mods.has_key(mod_imp_name):
# If we've already imported mod_imp_name, don't import it again
continue
# ignore modules that we've already determined aren't valid modules
if bad_mods.has_key(mod_imp_name):
continue
try:
# Auto-detect and load all FuncModules
blip = __import__("modules.%s" % (mod_imp_name), globals(),
locals(), [mod_imp_name])
for obj in dir(blip):
attr = getattr(blip, obj)
if isclass(attr) and issubclass(attr, func_module.FuncModule):
logger.debug("Loading %s module" % attr)
mods[mod_imp_name] = attr()
except ImportError, e:
# A module that raises an ImportError is (for now) simply not loaded.
errmsg = _("Could not load %s module: %s")
logger.warning(errmsg % (mod_imp_name, e))
bad_mods[mod_imp_name] = True
continue
except:
locals(), [mod_imp_name])
for obj in dir(blip):
attr = getattr(blip, obj)
if isclass(attr) and issubclass(attr, func_module.FuncModule):
logger.debug("Loading %s module" % attr)
mods[mod_imp_name] = attr()
except ImportError, e:
# A module that raises an ImportError is (for now) simply not loaded.
errmsg = _("Could not load %s module: %s")
logger.warning(errmsg % (mod_imp_name, e))
bad_mods[mod_imp_name] = True
continue
except:
errmsg = _("Could not load %s module")
logger.warning(errmsg % (mod_imp_name))
bad_mods[mod_imp_name] = True
continue
return mods
if __name__ == "__main__":
module_file_path = "/usr/lib/python2.5/site-packages/func/minion/modules/"
bar = module_walker(module_file_path)
print bar
for f in bar:
print f
print os.path.basename(f)
print os.path.split(f)
blip = __import__("modules.%s" % ( mod_imp_name), globals(), locals(), [mod_imp_name])
for obj in dir(blip):
attr = getattr(blip, obj)
if isclass(attr) and issubclass(attr, func_module.FuncModule):
logger.debug("Loading %s module" % attr)
mods[mod_imp_name] = attr()
except ImportError, e:
# A module that raises an ImportError is (for now) simply not loaded.
errmsg = _("Could not load %s module: %s")
logger.warning(errmsg % (mod_imp_name, e))
bad_mods[mod_imp_name] = True
continue
except:
errmsg = _("Could not load %s module")
logger.warning(errmsg % (mod_imp_name))
bad_mods[mod_imp_name] = True
continue
return mods
if __name__ == "__main__":
module_file_path = "/usr/lib/python2.5/site-packages/func/minion/modules/"
bar = module_walker(module_file_path)
print bar
for f in bar:
print f
print os.path.basename(f)
print os.path.split(f)