def __init__(self,
bert_config_file: str,
n_tags: List[str],
keep_prob: float,
attention_probs_keep_prob: float = None,
hidden_keep_prob: float = None,
encoder_layer_ids: List[int] = tuple(range(12)),
optimizer: str = None,
num_warmup_steps: int = None,
weight_decay_rate: float = 0.01,
return_probas: bool = False,
pretrained_bert: str = None,
min_learning_rate: float = 1e-06,
**kwargs) -> None:
super().__init__(**kwargs)
self.return_probas = return_probas
self.n_tags = n_tags
self.min_learning_rate = min_learning_rate
self.keep_prob = keep_prob
self.optimizer = optimizer
self.encoder_layer_ids = encoder_layer_ids
self.num_warmup_steps = num_warmup_steps
self.weight_decay_rate = weight_decay_rate
self.bert_config = BertConfig.from_json_file(
str(expand_path(bert_config_file)))
if attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - attention_probs_keep_prob
if hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - hidden_keep_prob
self.sess_config = tf.ConfigProto(allow_soft_placement=True)
self.sess_config.gpu_options.allow_growth = True
self.sess = tf.Session(config=self.sess_config)
self._init_graph()
self._init_optimizer()
self.sess.run(tf.global_variables_initializer())
if pretrained_bert is not None:
pretrained_bert = str(expand_path(pretrained_bert))
if tf.train.checkpoint_exists(pretrained_bert) \
and not tf.train.checkpoint_exists(str(self.load_path.resolve())):
logger.info('[initializing model with Bert from {}]'.format(
pretrained_bert))
# Exclude optimizer and classification variables from saved variables
var_list = self._get_saveable_variables(
exclude_scopes=('Optimizer', 'learning_rate', 'momentum',
'ner'))
saver = tf.train.Saver(var_list)
saver.restore(self.sess, pretrained_bert)
if self.load_path is not None:
self.load()
def __init__(self, bert_config_file, n_classes, keep_prob,
one_hot_labels=False, multilabel=False, return_probas=False,
attention_probs_keep_prob=None, hidden_keep_prob=None,
optimizer=None, num_warmup_steps=None, weight_decay_rate=0.01,
pretrained_bert=None, min_learning_rate=1e-06, **kwargs) -> None:
super().__init__(**kwargs)
self.return_probas = return_probas
self.n_classes = n_classes
self.min_learning_rate = min_learning_rate
self.keep_prob = keep_prob
self.one_hot_labels = one_hot_labels
self.multilabel = multilabel
self.optimizer = optimizer
self.num_warmup_steps = num_warmup_steps
self.weight_decay_rate = weight_decay_rate
if self.multilabel and not self.one_hot_labels:
raise RuntimeError('Use one-hot encoded labels for multilabel classification!')
if self.multilabel and not self.return_probas:
raise RuntimeError('Set return_probas to True for multilabel classification!')
self.bert_config = BertConfig.from_json_file(str(expand_path(bert_config_file)))
if attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - attention_probs_keep_prob
if hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - hidden_keep_prob
self.sess_config = tf.ConfigProto(allow_soft_placement=True)
self.sess_config.gpu_options.allow_growth = True
self.sess = tf.Session(config=self.sess_config)
self._init_graph()
self._init_optimizer()
self.sess.run(tf.global_variables_initializer())
if pretrained_bert is not None:
pretrained_bert = str(expand_path(pretrained_bert))
if tf.train.checkpoint_exists(pretrained_bert) \
and not (self.load_path and tf.train.checkpoint_exists(str(self.load_path.resolve()))):
logger.info('[initializing model with Bert from {}]'.format(pretrained_bert))
# Exclude optimizer and classification variables from saved variables
var_list = self._get_saveable_variables(
exclude_scopes=('Optimizer', 'learning_rate', 'momentum', 'output_weights', 'output_bias'))
saver = tf.train.Saver(var_list)
saver.restore(self.sess, pretrained_bert)
if self.load_path is not None:
self.load()
def __init__(self,
bert_config_file,
keep_prob=0.9,
attention_probs_keep_prob=None,
hidden_keep_prob=None,
optimizer=None,
weight_decay_rate=0.01,
pretrained_bert=None,
min_learning_rate=1e-06,
**kwargs) -> None:
super().__init__(**kwargs)
self.min_learning_rate = min_learning_rate
self.keep_prob = keep_prob
self.optimizer = optimizer
self.weight_decay_rate = weight_decay_rate
self.bert_config = BertConfig.from_json_file(
str(expand_path(bert_config_file)))
if attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - attention_probs_keep_prob
if hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - hidden_keep_prob
self.sess_config = tf.ConfigProto(allow_soft_placement=True)
self.sess_config.gpu_options.allow_growth = True
self.sess = tf.Session(config=self.sess_config)
self._init_graph()
self._init_optimizer()
if pretrained_bert is not None:
pretrained_bert = str(expand_path(pretrained_bert))
if tf.train.checkpoint_exists(pretrained_bert) \
and not tf.train.checkpoint_exists(str(self.load_path.resolve())):
logger.info('[initializing model with Bert from {}]'.format(
pretrained_bert))
# Exclude optimizer and classification variables from saved variables
var_list = self._get_saveable_variables(
exclude_scopes=('Optimizer', 'learning_rate', 'momentum',
'output_weights', 'output_bias'))
assignment_map = self.get_variables_to_restore(
var_list, pretrained_bert)
tf.train.init_from_checkpoint(pretrained_bert, assignment_map)
self.sess.run(tf.global_variables_initializer())
if self.load_path is not None:
self.load()
def __init__(self,
bert_config_file: str,
keep_prob: float,
attention_probs_keep_prob: Optional[float] = None,
hidden_keep_prob: Optional[float] = None,
optimizer: Optional[str] = None,
weight_decay_rate: Optional[float] = 0.01,
pretrained_bert: Optional[str] = None,
min_learning_rate: float = 1e-06,
**kwargs) -> None:
super().__init__(**kwargs)
self.min_learning_rate = min_learning_rate
self.keep_prob = keep_prob
self.optimizer = optimizer
self.weight_decay_rate = weight_decay_rate
self.bert_config = BertConfig.from_json_file(
str(expand_path(bert_config_file)))
if attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - attention_probs_keep_prob
if hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - hidden_keep_prob
self.sess_config = tf.ConfigProto(allow_soft_placement=True)
self.sess_config.gpu_options.allow_growth = True
self.sess = tf.Session(config=self.sess_config)
self._init_graph()
self._init_optimizer()
self.sess.run(tf.global_variables_initializer())
if pretrained_bert is not None:
pretrained_bert = str(expand_path(pretrained_bert))
if tf.train.checkpoint_exists(pretrained_bert) \
and not (self.load_path and tf.train.checkpoint_exists(str(self.load_path.resolve()))):
logger.info('[initializing model with Bert from {}]'.format(
pretrained_bert))
var_list = self._get_saveable_variables(
exclude_scopes=('Optimizer', 'learning_rate', 'momentum',
'squad'))
saver = tf.train.Saver(var_list)
saver.restore(self.sess, pretrained_bert)
if self.load_path is not None:
self.load()
def __init__(self, bert_config_file: str,
pretrained_bert: str,
vocab_file: str,
max_summary_length: int,
max_summary_length_in_tokens: Optional[bool] = False,
max_seq_length: Optional[int] = 128,
do_lower_case: Optional[bool] = False,
lang: Optional[str] = 'ru',
**kwargs) -> None:
self.max_summary_length = max_summary_length
self.max_summary_length_in_tokens = max_summary_length_in_tokens
self.bert_config = BertConfig.from_json_file(str(expand_path(bert_config_file)))
self.bert_preprocessor = BertPreprocessor(vocab_file=vocab_file, do_lower_case=do_lower_case,
max_seq_length=max_seq_length)
self.tokenize_reg = re.compile(r"[\w']+|[^\w ]")
if lang == 'ru':
from ru_sent_tokenize import ru_sent_tokenize
self.sent_tokenizer = ru_sent_tokenize
else:
from nltk import sent_tokenize
self.sent_tokenizer = sent_tokenize
self.sess_config = tf.ConfigProto(allow_soft_placement=True)
self.sess_config.gpu_options.allow_growth = True
self.sess = tf.Session(config=self.sess_config)
self._init_graph()
self.sess.run(tf.global_variables_initializer())
if pretrained_bert is not None:
pretrained_bert = str(expand_path(pretrained_bert))
if tf.train.checkpoint_exists(pretrained_bert):
logger.info('[initializing model with Bert from {}]'.format(pretrained_bert))
tvars = tf.trainable_variables()
assignment_map, _ = get_assignment_map_from_checkpoint(tvars, pretrained_bert)
tf.train.init_from_checkpoint(pretrained_bert, assignment_map)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import tensorflow as tf
from bert_dp.modeling import BertConfig, BertModel
from deeppavlov.models.preprocessors.bert_preprocessor import BertPreprocessor
bert_config = BertConfig.from_json_file(
'./bg_cs_pl_ru_cased_L-12_H-768_A-12/bert_config.json')
input_ids = tf.placeholder(shape=(None, None), dtype=tf.int32)
input_mask = tf.placeholder(shape=(None, None), dtype=tf.int32)
token_type_ids = tf.placeholder(shape=(None, None), dtype=tf.int32)
bert = BertModel(config=bert_config,
is_training=False,
input_ids=input_ids,
input_mask=input_mask,
token_type_ids=token_type_ids,
use_one_hot_embeddings=False)
preprocessor = BertPreprocessor(
vocab_file='./bg_cs_pl_ru_cased_L-12_H-768_A-12/vocab.txt',
do_lower_case=False,
max_seq_length=512)
with tf.Session() as sess:
# Load model
tf.train.Saver().restore(
def __init__(self,
n_tags: List[str],
keep_prob: float,
bert_config_file: str,
pretrained_bert: str = None,
attention_probs_keep_prob: float = None,
hidden_keep_prob: float = None,
use_crf=False,
encoder_layer_ids: List[int] = (-1, ),
optimizer: str = None,
weight_decay_rate: float = 1e-6,
use_birnn: bool = False,
birnn_cell_type: str = 'lstm',
birnn_hidden_size: int = 128,
ema_decay: float = None,
ema_variables_on_cpu: bool = True,
return_probas: bool = False,
freeze_embeddings: bool = False,
learning_rate: float = 1e-3,
bert_learning_rate: float = 2e-5,
min_learning_rate: float = 1e-07,
learning_rate_drop_patience: int = 20,
learning_rate_drop_div: float = 2.0,
load_before_drop: bool = True,
clip_norm: float = 1.0,
**kwargs) -> None:
super().__init__(
learning_rate=learning_rate,
learning_rate_drop_div=learning_rate_drop_div,
learning_rate_drop_patience=learning_rate_drop_patience,
load_before_drop=load_before_drop,
clip_norm=clip_norm,
**kwargs)
self.n_tags = n_tags
self.keep_prob = keep_prob
self.use_crf = use_crf
self.encoder_layer_ids = encoder_layer_ids
self.optimizer = optimizer
self.weight_decay_rate = weight_decay_rate
self.use_birnn = use_birnn
self.birnn_cell_type = birnn_cell_type
self.birnn_hidden_size = birnn_hidden_size
self.ema_decay = ema_decay
self.ema_variables_on_cpu = ema_variables_on_cpu
self.return_probas = return_probas
self.freeze_embeddings = freeze_embeddings
self.bert_learning_rate_multiplier = bert_learning_rate / learning_rate
self.min_learning_rate = min_learning_rate
self.bert_config = BertConfig.from_json_file(
str(expand_path(bert_config_file)))
if attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - attention_probs_keep_prob
if hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - hidden_keep_prob
self.sess_config = tf.ConfigProto(allow_soft_placement=True)
self.sess_config.gpu_options.allow_growth = True
self.sess = tf.Session(config=self.sess_config)
self._init_graph()
self._init_optimizer()
self.sess.run(tf.global_variables_initializer())
if pretrained_bert is not None:
pretrained_bert = str(expand_path(pretrained_bert))
if tf.train.checkpoint_exists(pretrained_bert) \
and not tf.train.checkpoint_exists(str(self.load_path.resolve())):
log.info('[initializing model with Bert from {}]'.format(
pretrained_bert))
# Exclude optimizer and classification variables from saved variables
var_list = self._get_saveable_variables(
exclude_scopes=('Optimizer', 'learning_rate', 'momentum',
'ner', 'EMA'))
saver = tf.train.Saver(var_list)
saver.restore(self.sess, pretrained_bert)
if self.load_path is not None:
self.load()
if self.ema:
self.sess.run(self.ema.init_op)
def __init__(self,
bert_config_file,
n_classes,
keep_prob,
batch_size,
num_ranking_samples,
one_hot_labels=False,
attention_probs_keep_prob=None,
hidden_keep_prob=None,
pretrained_bert=None,
resps=None,
resp_vecs=None,
resp_features=None,
resp_eval=True,
conts=None,
cont_vecs=None,
cont_features=None,
cont_eval=True,
bot_mode=0,
min_learning_rate=1e-06,
**kwargs) -> None:
super().__init__(**kwargs)
self.batch_size = batch_size
self.num_ranking_samples = num_ranking_samples
self.n_classes = n_classes
self.min_learning_rate = min_learning_rate
self.keep_prob = keep_prob
self.one_hot_labels = one_hot_labels
self.batch_size = batch_size
self.resp_eval = resp_eval
self.resps = resps
self.resp_vecs = resp_vecs
self.cont_eval = cont_eval
self.conts = conts
self.cont_vecs = cont_vecs
self.bot_mode = bot_mode
self.bert_config = BertConfig.from_json_file(
str(expand_path(bert_config_file)))
if attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - attention_probs_keep_prob
if hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - hidden_keep_prob
self.sess_config = tf.ConfigProto(allow_soft_placement=True)
self.sess_config.gpu_options.allow_growth = True
self.sess = tf.Session(config=self.sess_config)
self._init_graph()
self._init_optimizer()
self.sess.run(tf.global_variables_initializer())
if pretrained_bert is not None:
pretrained_bert = str(expand_path(pretrained_bert))
if tf.train.checkpoint_exists(pretrained_bert) \
and not tf.train.checkpoint_exists(str(self.load_path.resolve())):
logger.info('[initializing model with Bert from {}]'.format(
pretrained_bert))
# Exclude optimizer and classification variables from saved variables
var_list = self._get_saveable_variables(
exclude_scopes=('Optimizer', 'learning_rate', 'momentum',
'classification'))
saver = tf.train.Saver(var_list)
saver.restore(self.sess, pretrained_bert)
if self.load_path is not None:
self.load()
if self.resp_eval:
assert (self.resps is not None)
assert (self.resp_vecs is not None)
if self.cont_eval:
assert (self.conts is not None)
assert (self.cont_vecs is not None)
if self.resp_eval and self.cont_eval:
assert (len(self.resps) == len(self.conts))