def main(unused_argv):
if len(sys.argv) == 1:
flags._global_parser.print_help()
sys.exit(0)
# Loading model
m = model.load_model(FLAGS.dragnn_spec,
FLAGS.resource_path,
FLAGS.checkpoint_filename,
enable_tracing=FLAGS.enable_tracing,
tf_master=FLAGS.tf_master)
sess = m['session']
graph = m['graph']
builder = m['builder']
annotator = m['annotator']
# Analyze
# Prepare korean morphological analyzer for segmentation
from konlpy.tag import Komoran
komoran = Komoran()
startTime = time.time()
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
if not line: break
line = line.strip()
if not line: continue
segmented, tagged = model.segment_by_konlpy(line, komoran)
# ex) line = '제주 로 가다 는 비행기 가 심하다 는 비바람 에 회항 하 었 다 .'
line = ' '.join(segmented)
parsed_sentence = model.inference(sess, graph, builder, annotator,
line, FLAGS.enable_tracing)
out = model.parse_to_conll(parsed_sentence, tagged)
f = sys.stdout
f.write('# text = ' + line.encode('utf-8') + '\n')
for entry in out['conll']:
id = entry['id']
form = entry['form']
lemma = entry['lemma']
upostag = entry['upostag']
xpostag = entry['xpostag']
feats = entry['feats']
head = entry['head']
deprel = entry['deprel']
deps = entry['deps']
misc = entry['misc']
li = [
id, form, lemma, upostag, xpostag, feats, head, deprel, deps,
misc
]
f.write('\t'.join([str(e) for e in li]) + '\n')
f.write('\n\n')
durationTime = time.time() - startTime
sys.stderr.write("duration time = %f\n" % durationTime)
# Unloading model
model.unload_model(m)
def main(unused_argv) :
if len(sys.argv) == 1 :
flags._global_parser.print_help()
sys.exit(0)
# Loading model
m = model.load_model(FLAGS.dragnn_spec,
FLAGS.resource_path,
FLAGS.checkpoint_filename,
enable_tracing=FLAGS.enable_tracing,
tf_master=FLAGS.tf_master)
sess = m['session']
graph = m['graph']
builder = m['builder']
annotator = m['annotator']
# Analyze
startTime = time.time()
while 1 :
try : line = sys.stdin.readline()
except KeyboardInterrupt : break
if not line : break
line = line.strip()
if not line : continue
parsed_sentence = model.inference(sess, graph, builder, annotator, line, FLAGS.enable_tracing)
out = model.parse_to_conll(parsed_sentence)
f = sys.stdout
f.write('# text = ' + line.encode('utf-8') + '\n')
for entry in out['conll'] :
id = entry['id']
form = entry['form']
lemma = entry['lemma']
upostag = entry['upostag']
xpostag = entry['xpostag']
feats = entry['feats']
head = entry['head']
deprel = entry['deprel']
deps = entry['deps']
misc = entry['misc']
li = [id, form, lemma, upostag, xpostag, feats, head, deprel, deps, misc]
f.write('\t'.join([str(e) for e in li]) + '\n')
f.write('\n\n')
durationTime = time.time() - startTime
sys.stderr.write("duration time = %f\n" % durationTime)
# Unloading model
model.unload_model(m)
def test_model(raw_text):
segmentation_result = segmentation(raw_text) # return a dictionary
tokens = segmentation_result['segmentresult']
input_text = " ".join(
tokens) # because the dragnn is split by space in default
parsed_sentence = model.inference(dragnn_models['session'],
dragnn_models['graph'],
dragnn_models['builder'],
dragnn_models['annotator'], input_text,
False)
result = model.parse_to_conll(parsed_sentence)
return result
def get(self) :
start_time = time.time()
callback = self.get_argument('callback', '')
mode = self.get_argument('mode', 'product')
try :
# unicode
query = self.get_argument('q', '')
except :
query = "Invalid unicode in q"
debug = {}
debug['callback'] = callback
debug['mode'] = mode
pid = os.getpid()
debug['pid'] = pid
rst = {}
rst['msg'] = ''
rst['query'] = query
if mode == 'debug' : rst['debug'] = debug
try :
# convert to utf-8
query = query.encode('utf-8')
except :
rst['status'] = 500
rst['msg'] = "input query encode('utf-8') fail"
rst['output'] = []
else :
m = self.dragnn[pid]
sess = m['session']
graph = m['graph']
builder = m['builder']
annotator = m['annotator']
enable_tracing = self.enable_tracing
enable_konlpy = self.enable_konlpy
komoran = self.komoran
try :
out = {}
if enable_konlpy :
segmented, tagged = model.segment_by_konlpy(query, komoran)
query = ' '.join(segmented)
parsed_sentence = model.inference(sess, graph, builder, annotator, query, enable_tracing)
out = model.parse_to_conll(parsed_sentence, tagged)
else :
parsed_sentence = model.inference(sess, graph, builder, annotator, query, enable_tracing)
out = model.parse_to_conll(parsed_sentence)
out['text'] = query
rst['output'] = {}
except :
rst['status'] = 500
rst['msg'] = 'analyze() fail'
else :
rst['status'] = 200
rst['output'] = out
if mode == 'debug' :
duration_time = time.time() - start_time
debug['exectime'] = duration_time
try :
ret = json.dumps(rst,ensure_ascii=False,encoding="utf-8")
except :
msg = "json.dumps() fail for query %s" % (query)
self.log.debug(msg + "\n")
err = {}
err['status'] = 500
err['msg'] = msg
ret = json.dumps(err)
if mode == 'debug' :
self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0')
if callback.strip() :
self.set_header('Content-Type', 'application/javascript; charset=utf-8')
ret = 'if (typeof %s === "function") %s(%s);' % (callback, callback, ret)
else :
self.set_header('Content-Type', 'application/json; charset=utf-8')
self.write(ret)
self.finish()
builder = m['builder']
annotator = m['annotator']
enable_tracing = self.enable_tracing
enable_konlpy = self.enable_konlpy
komoran = self.komoran
# analyze line by line
out_list=[]
idx = 0
for line in content.split('\n') :
line = line.strip()
if not line : continue
if enable_konlpy :
segmented, tagged = model.segment_by_konlpy(line, komoran)
line = ' '.join(segmented)
parsed_sentence = model.inference(sess, graph, builder, annotator, line, enable_tracing)
out = model.parse_to_conll(parsed_sentence, tagged)
else :
parsed_sentence = model.inference(sess, graph, builder, annotator, line, enable_tracing)
out = model.parse_to_conll(parsed_sentence)
out['text'] = line
out_list.append(out)
idx += 1
self.write(dict(success=True, record=out_list, info=None))
self.finish()
class DragnnTestHandler(BaseHandler):
def post(self):
try :
content = self.get_argument('content', "", True)
content = content.encode('utf-8')