def __init__(self,model_file,Task=None,Searcher=None,
Updater=None,
beam_width=8,logger=None,cmd_args={},**conf):
"""
初始化
如果不设置，则读取已有模型。如果设置，就是学习新模型
"""
if logger==None :
logger=logging.getLogger(__name__)
console=logging.StreamHandler()
console.setLevel(logging.INFO)
logger.addHandler(console)
logger.setLevel(logging.INFO)
self.result_logger=logger
self.beam_width=beam_width#:搜索宽度
self.conf=conf
if model_file!=None:
file=gzip.open(model_file,"rb")
self.task=Task(model=pickle.load(file),logger=logger)
file.close()
else : # new model to train
self.paras=Parameters(Updater)
#self.paras=Parameters(Ada_Grad)
self.task=Task(logger=logger,paras=self.paras)
if hasattr(self.task,'init'):
self.task.init()
self.searcher=Searcher(self.task,beam_width)
self.step=0
def isan(**args):
orginal_args = args
ns = argparse.Namespace()
ns.logfile = '/dev/null'
for k, v in args.items():
setattr(ns, k, v)
args = ns
info_color = '34'
instream = sys.stdin if args.input == None else open(args.input, 'r')
outstream = sys.stdout if args.output == None else open(
args.output, 'a' if args.append else 'w')
rec = Recorder()
logger = logging.getLogger('s' + str(random.random()))
console = logging.StreamHandler()
logfile = logging.FileHandler(args.logfile, 'w')
logfile.setLevel(logging.DEBUG)
logfile.addFilter(ContextFilter())
recstream = logging.StreamHandler(rec)
console.setLevel(logging.INFO)
logger.addHandler(console)
logger.addHandler(logfile)
logger.addHandler(recstream)
if hasattr(args, 'log_handlers'):
for handler in args.log_handlers:
#handler.addFilter(ContextFilter())
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
if args.model_module:
mod, _, cls = args.model_module.rpartition('.')
Model = getattr(__import__(mod, globals(), locals(), [cls], 0), cls)
if args.task:
mod, _, cls = args.task.rpartition('.')
Task = getattr(__import__(mod, globals(), locals(), [cls], 0), cls)
if args.decoder:
mod, _, cls = args.decoder.rpartition('.')
Decoder = getattr(__import__(mod, globals(), locals(), [cls], 0), cls)
if args.updater:
mod, _, cls = args.updater.rpartition('.')
Updater = getattr(__import__(mod, globals(), locals(), [cls], 0), cls)
name_model = Model.name if hasattr(Model, 'name') else '给定学习算法'
name_decoder = Decoder.name if hasattr(Decoder, 'name') else '给定解码算法'
name_task = Task.name if hasattr(Task, 'name') else '给定任务算法'
name_updater = Updater.name if hasattr(Updater, 'name') else '某参数更新算法'
logger.info("""模型: %s 解码器: %s 搜索宽度: %s
任务: %s""" % (
make_color(name_model, info_color),
make_color(name_decoder, info_color),
make_color(args.beam_width, info_color),
make_color(name_task, info_color),
))
if args.train or args.append_model:
"""如果指定了训练集，就训练模型"""
logger.info(
"""参数更新算法 : %(updater)s batch size : %(bs)s""" % {
'bs': make_color(args.batch_size, info_color),
'updater': make_color(name_updater, info_color),
})
random.seed(args.seed)
model = Model(None, (lambda **x: Task(cmd_args=args, **x)),
Decoder,
beam_width=int(args.beam_width),
Updater=Updater,
logger=logger,
cmd_args=args)
if args.train:
logger.info('随机数种子: %s' % (make_color(str(args.seed))))
logger.info(
"由训练语料库%s迭代%s次，训练%s模型保存在%s。" %
(make_color(' '.join(args.train)), make_color(
args.iteration), name_task, make_color(args.model_file)))
if args.dev_file:
logger.info("开发集使用%s" % (make_color(' '.join(args.dev_file))))
model.train(args.train,
int(args.iteration),
peek=args.peek,
batch_size=args.batch_size,
dev_files=args.dev_file)
model.save(args.model_file)
if args.append_model: ### append multiple models
task = Task(cmd_args=args, paras=Parameters(Updater))
for m in args.append_model:
print(m)
task.add_model(pickle.load(gzip.open(m, 'rb')))
pickle.dump(task.dump_weights(), gzip.open(args.model_file, 'wb'))
if args.train and not args.test_file:
del logger
del model
return list(rec)
if not args.train:
print("使用模型文件%s进行%s" % (make_color(args.model_file), name_task),
file=sys.stderr)
#print(args.model_file)
model = Model(
args.model_file,
(lambda **x: Task(cmd_args=args, **x)),
Searcher=Decoder,
beam_width=int(args.beam_width),
logger=logger,
cmd_args=args,
)
"""如果指定了测试集，就测试模型"""
if args.test_file:
print("使用已经过%s的文件%s作为测试集" % (name_task, make_color(args.test_file)),
file=sys.stderr)
model.test(args.test_file)
return list(rec)
if not args.test_file and not args.append_model and not args.train:
threshold = args.threshold
print("以 %s 作为输入，以 %s 作为输出" %
(make_color('标准输入流'), make_color('标准输出流')),
file=sys.stderr)
if threshold:
print("输出分数差距在 %s 之内的候选词" % (make_color(threshold)),
file=sys.stderr)
for line in instream:
line = line.strip()
line = model.task.codec.decode(line)
raw = line.get('raw', '')
Y = line.get('Y_a', None)
if threshold:
print(model.task.codec.encode_candidates(
model(raw, Y, threshold=threshold)),
file=outstream)
else:
print(model.task.codec.encode(model(raw, Y)), file=outstream)
return list(rec)
