def load_lm(path, s3_path, model, model_class, quantized=False, **kwargs): check_file(path[model], s3_path[model], quantized=quantized, **kwargs) if quantized: model_path = 'quantized' else: model_path = 'model' g = load_graph(path[model][model_path], **kwargs) X = g.get_tensor_by_name('import/Placeholder:0') top_p = g.get_tensor_by_name('import/Placeholder_2:0') greedy = g.get_tensor_by_name('import/greedy:0') beam = g.get_tensor_by_name('import/beam:0') nucleus = g.get_tensor_by_name('import/nucleus:0') tokenizer = SentencePieceEncoder(path[model]['vocab']) return model_class( X=X, top_p=top_p, greedy=greedy, beam=beam, nucleus=nucleus, sess=generate_session(graph=g, **kwargs), tokenizer=tokenizer, )
def load(module, model, model_class, maxlen, quantized=False, **kwargs): path = check_file( file=model, module=module, keys={ 'model': 'model.pb', 'vocab': TRANSLATION_BPE_MODEL }, quantized=quantized, **kwargs, ) g = load_graph(path['model'], **kwargs) inputs = ['Placeholder'] outputs = ['logits'] input_nodes, output_nodes = nodes_session(g, inputs, outputs) encoder = SentencePieceEncoder(vocab_file=path['vocab']) return model_class( input_nodes=input_nodes, output_nodes=output_nodes, sess=generate_session(graph=g, **kwargs), encoder=encoder, maxlen=maxlen, )
def load_tatabahasa(module, model, model_class, quantized=False, **kwargs): path = check_file( file=model, module=module, keys={ 'model': 'model.pb', 'vocab': T2T_BPE_MODEL }, quantized=quantized, **kwargs, ) g = load_graph(path['model'], **kwargs) tokenizer = SentencePieceEncoder(vocab_file=path['vocab']) inputs = ['x_placeholder'] outputs = ['greedy', 'tag_greedy'] input_nodes, output_nodes = nodes_session(g, inputs, outputs) return model_class( input_nodes=input_nodes, output_nodes=output_nodes, sess=generate_session(graph=g, **kwargs), tokenizer=tokenizer, )
def load_lm(module, model, model_class, quantized=False, **kwargs): path = check_file( file=model, module=module, keys={ 'model': 'model.pb', 'vocab': T2T_BPE_MODEL }, quantized=quantized, **kwargs, ) g = load_graph(path['model'], **kwargs) X = g.get_tensor_by_name('import/Placeholder:0') top_p = g.get_tensor_by_name('import/Placeholder_2:0') greedy = g.get_tensor_by_name('import/greedy:0') beam = g.get_tensor_by_name('import/beam:0') nucleus = g.get_tensor_by_name('import/nucleus:0') tokenizer = SentencePieceEncoder(path['vocab']) inputs = ['Placeholder', 'Placeholder_2'] outputs = ['greedy', 'beam', 'nucleus'] input_nodes, output_nodes = nodes_session(g, inputs, outputs) return model_class( input_nodes=input_nodes, output_nodes=output_nodes, sess=generate_session(graph=g, **kwargs), tokenizer=tokenizer, )
def load_lm(path, s3_path, model, model_class, **kwargs): check_file(path[model], s3_path[model], **kwargs) g = load_graph(path[model]['model'], **kwargs) X = g.get_tensor_by_name('import/Placeholder:0') top_p = g.get_tensor_by_name('import/Placeholder_2:0') greedy = g.get_tensor_by_name('import/greedy:0') beam = g.get_tensor_by_name('import/beam:0') nucleus = g.get_tensor_by_name('import/nucleus:0') tokenizer = SentencePieceEncoder(path[model]['vocab']) return model_class( X, top_p, greedy, beam, nucleus, generate_session(graph=g, **kwargs), tokenizer, )
def load_tatabahasa(path, s3_path, model, model_class, quantized=False, **kwargs): check_file(path[model], s3_path[model], quantized=quantized, **kwargs) if quantized: model_path = 'quantized' else: model_path = 'model' g = load_graph(path[model][model_path], **kwargs) tokenizer = SentencePieceEncoder(path[model]['vocab']) return model_class( X=g.get_tensor_by_name('import/x_placeholder:0'), greedy=g.get_tensor_by_name('import/greedy:0'), tag_greedy=g.get_tensor_by_name('import/tag_greedy:0'), sess=generate_session(graph=g, **kwargs), tokenizer=tokenizer, )