def load_lm(path, s3_path, model, model_class, quantized=False, **kwargs):
check_file(path[model], s3_path[model], quantized=quantized, **kwargs)
if quantized:
model_path = 'quantized'
else:
model_path = 'model'
g = load_graph(path[model][model_path], **kwargs)
X = g.get_tensor_by_name('import/Placeholder:0')
top_p = g.get_tensor_by_name('import/Placeholder_2:0')
greedy = g.get_tensor_by_name('import/greedy:0')
beam = g.get_tensor_by_name('import/beam:0')
nucleus = g.get_tensor_by_name('import/nucleus:0')
tokenizer = SentencePieceEncoder(path[model]['vocab'])
return model_class(
X=X,
top_p=top_p,
greedy=greedy,
beam=beam,
nucleus=nucleus,
sess=generate_session(graph=g, **kwargs),
tokenizer=tokenizer,
)
def load(module, model, model_class, maxlen, quantized=False, **kwargs):
path = check_file(
file=model,
module=module,
keys={
'model': 'model.pb',
'vocab': TRANSLATION_BPE_MODEL
},
quantized=quantized,
**kwargs,
)
g = load_graph(path['model'], **kwargs)
inputs = ['Placeholder']
outputs = ['logits']
input_nodes, output_nodes = nodes_session(g, inputs, outputs)
encoder = SentencePieceEncoder(vocab_file=path['vocab'])
return model_class(
input_nodes=input_nodes,
output_nodes=output_nodes,
sess=generate_session(graph=g, **kwargs),
encoder=encoder,
maxlen=maxlen,
)
def load_tatabahasa(module, model, model_class, quantized=False, **kwargs):
path = check_file(
file=model,
module=module,
keys={
'model': 'model.pb',
'vocab': T2T_BPE_MODEL
},
quantized=quantized,
**kwargs,
)
g = load_graph(path['model'], **kwargs)
tokenizer = SentencePieceEncoder(vocab_file=path['vocab'])
inputs = ['x_placeholder']
outputs = ['greedy', 'tag_greedy']
input_nodes, output_nodes = nodes_session(g, inputs, outputs)
return model_class(
input_nodes=input_nodes,
output_nodes=output_nodes,
sess=generate_session(graph=g, **kwargs),
tokenizer=tokenizer,
)
def load_lm(module, model, model_class, quantized=False, **kwargs):
path = check_file(
file=model,
module=module,
keys={
'model': 'model.pb',
'vocab': T2T_BPE_MODEL
},
quantized=quantized,
**kwargs,
)
g = load_graph(path['model'], **kwargs)
X = g.get_tensor_by_name('import/Placeholder:0')
top_p = g.get_tensor_by_name('import/Placeholder_2:0')
greedy = g.get_tensor_by_name('import/greedy:0')
beam = g.get_tensor_by_name('import/beam:0')
nucleus = g.get_tensor_by_name('import/nucleus:0')
tokenizer = SentencePieceEncoder(path['vocab'])
inputs = ['Placeholder', 'Placeholder_2']
outputs = ['greedy', 'beam', 'nucleus']
input_nodes, output_nodes = nodes_session(g, inputs, outputs)
return model_class(
input_nodes=input_nodes,
output_nodes=output_nodes,
sess=generate_session(graph=g, **kwargs),
tokenizer=tokenizer,
)
def load_lm(path, s3_path, model, model_class, **kwargs):
check_file(path[model], s3_path[model], **kwargs)
g = load_graph(path[model]['model'], **kwargs)
X = g.get_tensor_by_name('import/Placeholder:0')
top_p = g.get_tensor_by_name('import/Placeholder_2:0')
greedy = g.get_tensor_by_name('import/greedy:0')
beam = g.get_tensor_by_name('import/beam:0')
nucleus = g.get_tensor_by_name('import/nucleus:0')
tokenizer = SentencePieceEncoder(path[model]['vocab'])
return model_class(
X,
top_p,
greedy,
beam,
nucleus,
generate_session(graph=g, **kwargs),
tokenizer,
)
def load_tatabahasa(path,
s3_path,
model,
model_class,
quantized=False,
**kwargs):
check_file(path[model], s3_path[model], quantized=quantized, **kwargs)
if quantized:
model_path = 'quantized'
else:
model_path = 'model'
g = load_graph(path[model][model_path], **kwargs)
tokenizer = SentencePieceEncoder(path[model]['vocab'])
return model_class(
X=g.get_tensor_by_name('import/x_placeholder:0'),
greedy=g.get_tensor_by_name('import/greedy:0'),
tag_greedy=g.get_tensor_by_name('import/tag_greedy:0'),
sess=generate_session(graph=g, **kwargs),
tokenizer=tokenizer,
)