def load(self, fname=None):
if fname is not None:
self.load_path = fname
if self.pretrained_bert:
log.info(f"From pretrained {self.pretrained_bert}.")
config = AutoConfig.from_pretrained(self.pretrained_bert,
num_labels=self.n_classes,
output_attentions=False,
output_hidden_states=False)
self.model = AutoModelForSequenceClassification.from_pretrained(
self.pretrained_bert, config=config)
elif self.bert_config_file and Path(self.bert_config_file).is_file():
self.bert_config = AutoConfig.from_json_file(
str(expand_path(self.bert_config_file)))
if self.attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - self.attention_probs_keep_prob
if self.hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - self.hidden_keep_prob
self.model = AutoModelForSequenceClassification.from_config(
config=self.bert_config)
else:
raise ConfigError("No pre-trained BERT model is given.")
self.model.to(self.device)
self.optimizer = getattr(torch.optim, self.optimizer_name)(
self.model.parameters(), **self.optimizer_parameters)
if self.lr_scheduler_name is not None:
self.lr_scheduler = getattr(torch.optim.lr_scheduler,
self.lr_scheduler_name)(
self.optimizer,
**self.lr_scheduler_parameters)
if self.load_path:
log.info(f"Load path {self.load_path} is given.")
if isinstance(self.load_path,
Path) and not self.load_path.parent.is_dir():
raise ConfigError("Provided load path is incorrect!")
weights_path = Path(self.load_path.resolve())
weights_path = weights_path.with_suffix(f".pth.tar")
if weights_path.exists():
log.info(f"Load path {weights_path} exists.")
log.info(
f"Initializing `{self.__class__.__name__}` from saved.")
# now load the weights, optimizer from saved
log.info(f"Loading weights from {weights_path}.")
checkpoint = torch.load(weights_path, map_location=self.device)
self.model.load_state_dict(checkpoint["model_state_dict"])
self.optimizer.load_state_dict(
checkpoint["optimizer_state_dict"])
self.epochs_done = checkpoint.get("epochs_done", 0)
else:
log.info(
f"Init from scratch. Load path {weights_path} does not exist."
)
def __call__(self, batch: List[List[str]], tags_batch: Optional[List[List[str]]] = None, mean: bool = None,
*args, **kwargs) -> List[Union[list, np.ndarray]]:
"""
Infer on the given data
Args:
batch: tokenized text samples
tags_batch: optional batch of corresponding tags
mean: whether to return mean token embedding (does not depend on self.mean)
*args: additional arguments
**kwargs: additional arguments
Returns:
"""
if self.tags_vocab:
if tags_batch is None:
raise ConfigError("TfidfWeightedEmbedder got 'tags_vocab_path' but __call__ did not get tags_batch.")
batch = [self._tags_encode(sample, tags_sample, mean=mean) for sample, tags_sample in zip(batch, tags_batch)]
else:
if tags_batch:
raise ConfigError("TfidfWeightedEmbedder got tags batch, but 'tags_vocab_path' is empty.")
batch = [self._encode(sample, mean=mean) for sample in batch]
if self.pad_zero:
batch = zero_pad(batch)
return batch
def __init__(self, **kwargs):
"""
Initialize and train vocabularies, initializes embedder, tokenizer,
and then initialize model using parameters from opt dictionary (from config),
if model is being initialized from saved
Args:
model_path: path to model serialization dir or file.
It is always an empty string and is ignored if it is not set in json config.
model_dir: name of a serialization dir, can be default or set in json config
model_file: name of a serialization file (usually binary model file),
can be default or set in json config
embedder: instance of FasttextEmbedder class
tokenizer: instance of NLTKTokenizer class
**kwargs:
"""
super().__init__(**kwargs) # self.opt initialized in here
self.tokenizer = self.opt.pop('tokenizer')
self.fasttext_model = self.opt.pop('embedder')
self.classes = list(np.sort(np.array(list(self.opt.get('classes')))))
self.opt['classes'] = self.classes
self.n_classes = len(self.classes)
if self.n_classes == 0:
ConfigError("Please, provide vocabulary with considered intents.")
self.opt['embedding_size'] = self.fasttext_model.dim
if self.fasttext_model.load_path:
current_fasttext_md5 = md5_hashsum([self.fasttext_model.load_path])
# Parameters required to init model
params = {
"model_name": self.opt.get('model_name'),
"optimizer_name": self.opt.get('optimizer'),
"loss_name": self.opt.get('loss'),
"lear_rate": self.opt.get('lear_rate'),
"lear_rate_decay": self.opt.get('lear_rate_decay')
}
self.model = self.load(**params)
self._init_missed_params()
self._change_not_fixed_params(**kwargs)
# Check if md5 hash sum of current loaded fasttext model
# is equal to saved
try:
self.opt['fasttext_md5']
except KeyError:
self.opt['fasttext_md5'] = current_fasttext_md5
else:
if self.opt['fasttext_md5'] != current_fasttext_md5:
raise ConfigError(
"Given fasttext model does NOT match fasttext model used previously to train loaded model"
)
print("Model was successfully initialized!\nModel summary:\n{}".format(
self.model.summary()))
def from_params(params: Dict, mode: str = 'infer', **kwargs) -> Component:
"""Builds and returns the Component from corresponding dictionary of parameters."""
# what is passed in json:
config_params = {k: _resolve(v) for k, v in params.items()}
# get component by reference (if any)
if 'ref' in config_params:
try:
return _refs[config_params['ref']]
except KeyError:
e = ConfigError(
'Component with id "{id}" was referenced but not initialized'.
format(id=config_params['ref']))
log.exception(e)
raise e
elif 'config_path' in config_params:
from deeppavlov.core.commands.infer import build_model_from_config
deeppavlov_root = get_deeppavlov_root()
refs = _refs.copy()
_refs.clear()
config = read_json(expand_path(config_params['config_path']))
model = build_model_from_config(config)
set_deeppavlov_root({'deeppavlov_root': deeppavlov_root})
_refs.clear()
_refs.update(refs)
return model
elif 'class' in config_params:
cls = cls_from_str(config_params.pop('class'))
else:
cls_name = config_params.pop('name', None)
if not cls_name:
e = ConfigError(
'Component config has no `name` nor `ref` or `class` fields')
log.exception(e)
raise e
cls = get_model(cls_name)
# find the submodels params recursively
config_params = {k: _init_param(v, mode) for k, v in config_params.items()}
try:
spec = inspect.getfullargspec(cls)
if 'mode' in spec.args + spec.kwonlyargs or spec.varkw is not None:
kwargs['mode'] = mode
component = cls(**dict(config_params, **kwargs))
try:
_refs[config_params['id']] = component
except KeyError:
pass
except Exception:
log.exception("Exception in {}".format(cls))
raise
return component
def append(self, component: Union[Component, FunctionType], in_x: [str, list, dict] = None,
out_params: [str, list] = None, in_y: [str, list, dict] = None, main: bool = False):
if isinstance(in_x, str):
in_x = [in_x]
if isinstance(in_y, str):
in_y = [in_y]
if isinstance(out_params, str):
out_params = [out_params]
in_x = in_x or self.in_x
if isinstance(in_x, dict):
x_keys, in_x = zip(*in_x.items())
else:
x_keys = []
out_params = out_params or in_x
if in_y is not None:
if isinstance(in_y, dict):
y_keys, in_y = zip(*in_y.items())
else:
y_keys = []
keys = x_keys + y_keys
if bool(x_keys) != bool(y_keys):
raise ConfigError('`in` and `in_y` for a component have to both be lists or dicts')
component: NNModel
main = True
assert self.train_map.issuperset(in_x + in_y), ('Arguments {} are expected but only {} are set'
.format(in_x + in_y, self.train_map))
preprocessor = Chainer(self.in_x, in_x + in_y, self.in_y)
for (t_in_x_keys, t_in_x), t_out, t_component in self.train_pipe:
if t_in_x_keys:
t_in_x = dict(zip(t_in_x_keys, t_in_x))
preprocessor.append(t_component, t_in_x, t_out)
def train_on_batch(*args, **kwargs):
preprocessed = preprocessor.compute(*args, **kwargs)
if len(in_x + in_y) == 1:
preprocessed = [preprocessed]
if keys:
return component.train_on_batch(**dict(zip(keys, preprocessed)))
else:
return component.train_on_batch(*preprocessed)
self.train_on_batch = train_on_batch
self.process_event = component.process_event
if main:
self.main = component
if self.forward_map.issuperset(in_x):
self.pipe.append(((x_keys, in_x), out_params, component))
self.forward_map = self.forward_map.union(out_params)
if self.train_map.issuperset(in_x):
self.train_pipe.append(((x_keys, in_x), out_params, component))
self.train_map = self.train_map.union(out_params)
else:
raise ConfigError('Arguments {} are expected but only {} are set'.format(in_x, self.train_map))
def _load(self, model_name: str) -> None:
"""
Initialize uncompiled model from saved params and weights
Args:
model_name: name of model function described as a method of this class
Returns:
model with loaded weights and network parameters from files
but compiled with given learning parameters
"""
if self.load_path:
if isinstance(self.load_path,
Path) and not self.load_path.parent.is_dir():
raise ConfigError("Provided load path is incorrect!")
opt_path = Path("{}_opt.json".format(str(
self.load_path.resolve())))
weights_path = Path("{}.h5".format(str(self.load_path.resolve())))
if opt_path.exists() and weights_path.exists():
log.info("[initializing `{}` from saved]".format(
self.__class__.__name__))
self.opt["final_learning_rate"] = read_json(opt_path).get(
"final_learning_rate")
model_func = getattr(self, model_name, None)
if callable(model_func):
model = model_func(**self.opt)
else:
raise AttributeError(
"Model {} is not defined".format(model_name))
log.info("[loading weights from {}]".format(weights_path.name))
try:
model.load_weights(str(weights_path))
except ValueError:
raise ConfigError(
"Some non-changeable parameters of neural network differ"
" from given pre-trained model")
self.model = model
return None
else:
self.model = self.init_model_from_scratch(model_name)
return None
else:
log.warning("No `load_path` is provided for {}".format(
self.__class__.__name__))
self.model = self.init_model_from_scratch(model_name)
return None
def from_params(params: Dict, **kwargs) -> Component:
# what is passed in json:
config_params = {k: _resolve(v) for k, v in params.items()}
# get component by reference (if any)
if 'ref' in config_params:
try:
return _refs[config_params['ref']]
except KeyError:
e = ConfigError(
'Component with id "{id}" was referenced but not initialized'.
format(id=config_params['ref']))
log.exception(e)
raise e
try:
cls_name = config_params.pop('name')
except KeyError:
e = ConfigError('Component config has no `name` nor `ref` fields')
log.exception(e)
raise e
try:
cls = REGISTRY[cls_name]
except KeyError:
e = ConfigError('Class {} is not registered.'.format(cls_name))
log.exception(e)
raise e
# find the submodels params recursively
for param_name, subcls_params in config_params.items():
if isinstance(subcls_params, dict):
if 'name' not in subcls_params and 'ref' not in subcls_params:
"This parameter is passed as dict to the class constructor."
" The user didn't intent it to be a component."
for k, v in subcls_params.items():
subcls_params[k] = _resolve(v)
continue
config_params[param_name] = from_params(subcls_params,
vocabs=kwargs['vocabs'],
mode=kwargs['mode'])
try:
component = cls(**dict(config_params, **kwargs))
try:
_refs[config_params['id']] = component
except KeyError:
pass
except Exception:
log.exception("Exception in {}".format(cls))
raise
return component
def load(self) -> None:
if self.load_path:
if self.load_path.is_file():
log.info(f"[loading vocabulary from {self.load_path}]")
self.tokens = []
for ln in self.load_path.open('r', encoding='utf8'):
token = ln.strip().split()[0]
self.tokens.append(token)
else:
raise ConfigError(f"Provided `load_path` for {self.__class__.__name__} doesn't exist!")
else:
raise ConfigError(f"`load_path` for {self} is not provided!")
def __init__(self,
embedder: Component,
tokenizer: Component = None,
pad_zero: bool = False,
mean: bool = False,
tags_vocab_path: str = None,
vectorizer: Component = None,
counter_vocab_path: str = None,
idf_base_count: int = 100,
log_base: int = 10,
min_idf_weight=0.0,
**kwargs) -> None:
"""
Initialize embedder with given parameters.
"""
self.embedder = embedder
self.dim = self.embedder.dim
self.mean = mean
self.pad_zero = pad_zero
if tokenizer is None:
self.tokenizer = self.space_detokenizer
else:
self.tokenizer = tokenizer
if vectorizer and counter_vocab_path:
raise ConfigError(
"TfidfWeightedEmbedder got vectorizer and counter_vocab_path simultaneously."
" Remove one of them, please")
elif vectorizer:
self.vectorizer = vectorizer
self.vocabulary = np.array(
self.vectorizer.model.get_feature_names())
elif counter_vocab_path:
self.counter_vocab_path = expand_path(counter_vocab_path)
self.counter_vocab, self.min_count = self.load_counter_vocab(
self.vocab_path)
self.idf_base_count = idf_base_count
self.log_base = log_base
self.min_idf_weight = min_idf_weight
else:
raise ConfigError(
"TfidfWeightedEmbedder did not get vectorizer or counter_vocab_path."
" Set one of them, please")
if tags_vocab_path:
self.tags_vocab = self.load_tags_vocab(
expand_path(tags_vocab_path))
else:
self.tags_vocab = None
def __init__(self, depth: int, pad_zeros: bool = False,
single_vector=False, *args, **kwargs):
self._depth = depth
self._pad_zeros = pad_zeros
self.single_vector = single_vector
if self._pad_zeros and self.single_vector:
raise ConfigError("Cannot perform ``single_vector`` with zero padding for OneHotter")
def append(self, in_x, out_params, component, in_y=None, main=False):
if in_y is not None:
main = True
assert self.train_map.issuperset(in_x + in_y), (
'Arguments {} are expected but only {} are set'.format(
in_x + in_y, self.train_map))
preprocessor = Chainer(self.in_x, in_x + in_y, self.in_y)
for t_in_x, t_out, t_component in self.train_pipe:
preprocessor.append(t_in_x, t_out, t_component)
def train_on_batch(*args, **kwargs):
preprocessed = preprocessor(*args, **kwargs)
return component.train_on_batch(*preprocessed)
self.train_on_batch = train_on_batch
if main:
self.main = component
if self.forward_map.issuperset(in_x):
self.pipe.append((in_x, out_params, component))
self.forward_map = self.forward_map.union(out_params)
if self.train_map.issuperset(in_x):
self.train_pipe.append((in_x, out_params, component))
self.train_map = self.train_map.union(out_params)
else:
raise ConfigError(
'Arguments {} are expected but only {} are set'.format(
in_x, self.train_map))
def __init__(
self,
n_classes: int,
model_name: str,
num_ner_tags: int,
pretrained_bert: str = None,
criterion: str = "CrossEntropyLoss",
optimizer: str = "AdamW",
optimizer_parameters: Dict = None,
return_probas: bool = False,
attention_probs_keep_prob: Optional[float] = None,
hidden_keep_prob: Optional[float] = None,
clip_norm: Optional[float] = None,
threshold: Optional[float] = None,
device: str = "cpu",
**kwargs
) -> None:
"""
Transformer-based model on PyTorch for relation extraction. It predicts a relation hold between entities in a
text sample (one or several sentences).
Args:
n_classes: number of output classes
model_name: the model which will be used for extracting the relations
num_ner_tags: number of NER tags
pretrained_bert: key title of pretrained Bert model (e.g. "bert-base-uncased")
criterion: criterion name from `torch.nn`
optimizer: optimizer name from `torch.optim`
optimizer_parameters: dictionary with optimizer's parameters
return_probas: set this to `True` if you need the probabilities instead of raw answers
attention_probs_keep_prob: keep_prob for Bert self-attention layers
hidden_keep_prob: keep_prob for Bert hidden layers
clip_norm: clip gradients by norm
threshold: manually set value for defining the positively predicted classes (instead of adaptive one)
device: cpu/gpu device to use for training the model
"""
self.n_classes = n_classes
self.num_ner_tags = num_ner_tags
self.pretrained_bert = pretrained_bert
self.return_probas = return_probas
self.attention_probs_keep_prob = attention_probs_keep_prob
self.hidden_keep_prob = hidden_keep_prob
self.clip_norm = clip_norm
self.threshold = threshold
self.device = device
if self.n_classes == 0:
raise ConfigError("Please provide a valid number of classes.")
if optimizer_parameters is None:
optimizer_parameters = {"lr": 5e-5, "weight_decay": 0.01, "eps": 1e-6}
super().__init__(
n_classes=n_classes,
model_name=model_name,
optimizer=optimizer,
criterion=criterion,
optimizer_parameters=optimizer_parameters,
return_probas=return_probas,
device=self.device,
**kwargs)
def init_from_scratch(self) -> None:
"""
Initialize ``self.model`` as some sklearn model from scratch with given in ``self.model_params`` parameters.
Returns:
None
"""
log.info("Initializing model {} from scratch".format(self.model_class))
model_function = cls_from_str(self.model_class)
if model_function is None:
raise ConfigError(
"Model with {} model_class was not found.".format(
self.model_class))
given_params = {}
if self.model_params:
available_params = self.get_function_params(model_function)
for param_name in self.model_params.keys():
if param_name in available_params:
try:
given_params[param_name] = cls_from_str(
self.model_params[param_name])
except (AttributeError, ValueError, ConfigError):
given_params[param_name] = self.model_params[
param_name]
self.model = model_function(**given_params)
return
def __init__(self,
optimizer: str = 'AdamOptimizer',
clip_norm: float = None,
momentum: float = None,
**kwargs) -> None:
TFModel.__init__(self, **kwargs)
try:
self._optimizer = cls_from_str(optimizer)
except Exception:
self._optimizer = getattr(tf.train, optimizer.split(':')[-1])
if not issubclass(self._optimizer, tf.train.Optimizer):
raise ConfigError("`optimizer` should be tensorflow.train.Optimizer subclass")
self._clip_norm = clip_norm
if (momentum is None) and\
self._optimizer not in (tf.train.AdagradOptimizer,
tf.train.AdagradOptimizer,
tf.train.GradientDescentOptimizer,
tf.train.ProximalGradientDescentOptimizer,
tf.train.ProximalAdagradOptimizer):
momentum = 0.9
kwargs['momentum'] = momentum
LRScheduledModel.__init__(self, **kwargs)
def save(self, fname=None):
"""
Save the model parameters into <<fname>>_opt.json (or <<ser_file>>_opt.json)
and model weights into <<fname>>.h5 (or <<ser_file>>.h5)
Args:
fname: file_path to save model. If not explicitly given seld.opt["ser_file"] will be used
Returns:
None
"""
if not fname:
fname = self.save_path
else:
fname = Path(fname).resolve()
if not fname.parent.is_dir():
raise ConfigError("Provided save path is incorrect!")
else:
opt_path = f"{fname}_opt.json"
weights_path = f"{fname}.h5"
log.info(f"[saving model to {opt_path}]")
self.model.save_weights(weights_path)
save_json(self.opt, opt_path)
return True
def _load_nn_params(self) -> None:
if self.debug:
log.debug(f"BEFORE {self.__class__.__name__} _load_nn_params()")
path = str(self.load_path.with_suffix('.json').resolve())
if self.debug:
log.debug(
f"INSIDE {self.__class__.__name__} _load_nn_params(): path={path}"
)
# log.info(f"[loading parameters from {path}]")
with open(path, 'r', encoding='utf8') as fp:
params = json.load(fp)
if self.debug:
log.debug(f"INSIDE {self.__class__.__name__} _load_nn_params(): "
f"params={params}, GRAPH_PARAMS={self.GRAPH_PARAMS}")
for p in self.GRAPH_PARAMS:
if self.__getattribute__(p) != params.get(p) and p not in {
'attn', 'attention_mechanism', 'attention_params'
}:
# todo backward-compatible attention serialization
raise ConfigError(
f"`{p}` parameter must be equal to saved"
f" model parameter value `{params.get(p)}`,"
f" but is equal to `{self.__getattribute__(p)}`")
if self.debug:
log.debug(f"AFTER {self.__class__.__name__} _load_nn_params()")
def pad_texts(
self, sentences: List[List[np.ndarray]]
) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:
"""
Cut and pad tokenized texts to self.opt["text_size"] tokens
Args:
sentences: list of lists of tokens
Returns:
array of embedded texts
"""
pad = np.zeros(self.opt['embedding_size'])
cut_batch = [sen[:self.opt['text_size']] for sen in sentences]
if self.opt["padding"] == "pre":
cut_batch = [[pad] * (self.opt['text_size'] - len(tokens)) +
list(tokens) for tokens in cut_batch]
elif self.opt["padding"] == "post":
cut_batch = [
list(tokens) + [pad] * (self.opt['text_size'] - len(tokens))
for tokens in cut_batch
]
else:
raise ConfigError("Padding type {} is not acceptable".format(
self.opt['padding']))
return np.asarray(cut_batch)
def load(self):
if self.load_path:
if self.load_path.is_file():
log.info("[loading vocabulary from {}]".format(self.load_path))
tokens, counts = [], []
for ln in self.load_path.open('r'):
token, cnt = ln.split('\t', 1)
tokens.append(token)
counts.append(int(cnt))
self._train(tokens=tokens, counts=counts, update=True)
elif isinstance(self.load_path, Path):
if not self.load_path.parent.is_dir():
raise ConfigError("Provided `load_path` for {} doesn't exist!".format(
self.__class__.__name__))
else:
raise ConfigError("`load_path` for {} is not provided!".format(self))
def load(self):
if self.load_path:
if self.load_path.is_file():
print("[loading error_model from `{}`]".format(self.load_path))
with open(self.load_path, 'r', newline='') as tsv_file:
reader = csv.reader(tsv_file, delimiter='\t')
for w, s, p in reader:
self.costs[(w, s)] = log(float(p))
elif isinstance(self.load_path, Path):
if not self.load_path.parent.is_dir():
raise ConfigError(
"Provided `load_path` for {} doesn't exist!".format(
self.__class__.__name__))
else:
raise ConfigError(
"`load_path` for {} is not provided!".format(self))
def save(self, fname: str = None) -> None:
"""
Save the model parameters into <<fname>>_opt.json (or <<ser_file>>_opt.json)
and model weights into <<fname>>.h5 (or <<ser_file>>.h5)
Args:
fname: file_path to save model. If not explicitly given seld.opt["ser_file"] will be used
Returns:
None
"""
if not fname:
fname = self.save_path
else:
fname = Path(fname).resolve()
if not fname.parent.is_dir():
raise ConfigError("Provided save path is incorrect!")
else:
opt_path = f"{fname}_opt.json"
weights_path = f"{fname}.h5"
log.info(f"[saving model to {opt_path}]")
self.model.save_weights(weights_path)
# if model was loaded from one path and saved to another one
# then change load_path to save_path for config
self.opt["epochs_done"] = self.epochs_done
self.opt["final_learning_rate"] = K.eval(self.optimizer.lr) / (1. +
K.eval(self.optimizer.decay) * self.batches_seen)
if self.opt.get("load_path") and self.opt.get("save_path"):
if self.opt.get("save_path") != self.opt.get("load_path"):
self.opt["load_path"] = str(self.opt["save_path"])
save_json(self.opt, opt_path)
def from_params(cls: Type, params: Dict, **kwargs) -> Type['T']:
# what is passed in json:
config_params = {k: v for k, v in params.items() if k != 'name'}
# find the submodels params recursively
for param_name, subcls_params in config_params.items():
if isinstance(subcls_params, dict):
try:
subcls_name = subcls_params['name']
except KeyError:
"This parameter is passed as dict to the class constructor."
" The user didn't intent it to be a model."
continue
try:
subcls = REGISTRY[subcls_name]
except KeyError:
raise ConfigError(
"The class {} is not registered. Either register this class,"
" or rename the parameter.".format(
subcls_params['name']))
config_params[param_name] = from_params(subcls, subcls_params,
vocabs=kwargs['vocabs'],
mode=kwargs['mode'])
try:
model = cls(**dict(config_params, **kwargs))
except Exception:
print("Exception in {}".format(cls), file=sys.stderr)
raise
return model
def __call__(self, data: Union[np.ndarray, List[List[float]],
List[List[int]]], *args,
**kwargs) -> Union[List[List[str]], List[str]]:
"""
Process probabilities to labels
Args:
data: list of vectors with probability distribution
*args:
**kwargs:
Returns:
list of labels (only label classification) or list of lists of labels (multi-label classification)
"""
if self.confident_threshold:
return [
list(np.where(np.array(d) > self.confident_threshold)[0])
for d in data
]
elif self.max_proba:
return [[np.argmax(d)] for d in data]
elif self.top_n:
return [np.argsort(d)[::-1][:self.top_n] for d in data]
else:
raise ConfigError(
"Proba2Labels requires one of three arguments: bool `max_proba` or "
"float `confident_threshold` for multi-label classification or"
"integer `top_n` for choosing several labels with the highest probabilities"
)
def _init_params(self):
self.learning_rate = self.opt['learning_rate']
self.end_learning_rate = self.opt['end_learning_rate']
self.decay_steps = self.opt['decay_steps']
self.decay_power = self.opt['decay_power']
self.dropout_rate = self.opt['dropout_rate']
self.hidden_size = self.opt['hidden_size']
self.action_size = self.opt['action_size']
self.obs_size = self.opt['obs_size']
self.dense_size = self.opt['dense_size']
self.l2_reg = self.opt['l2_reg_coef']
self._optimizer = None
if hasattr(tf.train, self.opt['optimizer']):
self._optimizer = getattr(tf.train, self.opt['optimizer'])
if not issubclass(self._optimizer, tf.train.Optimizer):
raise ConfigError("`optimizer` parameter should be a name of"
" tf.train.Optimizer subclass")
attn = self.opt.get('attention_mechanism')
if attn:
self.opt['attention_mechanism'] = attn
self.attn = \
collections.namedtuple('attention_mechanism', attn.keys())(**attn)
self.obs_size -= attn['token_size']
else:
self.attn = None
def get_metrics_by_names(names: list) -> List[Callable[..., Any]]:
"""Returns a list of metric callables with corresponding names."""
not_found = [name for name in names if name not in _REGISTRY]
if not_found:
raise ConfigError(
'Names {} are not registered as metrics'.format(not_found))
return [fn_from_str(_REGISTRY[name]) for name in names]
def read_data_by_config(config: dict):
"""Read data by dataset_reader from specified config."""
dataset_config = config.get('dataset', None)
if dataset_config:
config.pop('dataset')
ds_type = dataset_config['type']
if ds_type == 'classification':
reader = {'class_name': 'basic_classification_reader'}
iterator = {'class_name': 'basic_classification_iterator'}
config['dataset_reader'] = {**dataset_config, **reader}
config['dataset_iterator'] = {**dataset_config, **iterator}
else:
raise Exception("Unsupported dataset type: {}".format(ds_type))
try:
reader_config = dict(config['dataset_reader'])
except KeyError:
raise ConfigError("No dataset reader is provided in the JSON config.")
reader = get_model(reader_config.pop('class_name'))()
data_path = reader_config.pop('data_path', '')
if isinstance(data_path, list):
data_path = [expand_path(x) for x in data_path]
else:
data_path = expand_path(data_path)
return reader.read(data_path, **reader_config)
def get_model(name: str) -> type:
"""Returns a registered class object with the name given in the string."""
if name not in _REGISTRY:
if ':' not in name:
raise ConfigError("Model {} is not registered.".format(name))
return cls_from_str(name)
return cls_from_str(_REGISTRY[name])
def save(self, fname: Optional[str] = None, *args, **kwargs) -> None:
"""Save torch model to `fname` (if `fname` is not given, use `self.save_path`). Checkpoint includes
`model_state_dict`, `optimizer_state_dict`, and `epochs_done` (number of training epochs).
Args:
fname:
*args:
**kwargs:
Returns:
"""
if fname is None:
fname = self.save_path
if not fname.parent.is_dir():
raise ConfigError("Provided save path is incorrect!")
weights_path = Path(fname).with_suffix(f".pth.tar")
log.info(f"Saving model to {weights_path}.")
# move the model to `cpu` before saving to provide consistency
torch.save({
"model_state_dict": self.model.cpu().state_dict(),
"optimizer_state_dict": self.optimizer.state_dict(),
"epochs_done": self.epochs_done
}, weights_path)
# return it back to device (necessary if it was on `cuda`)
self.model.to(self.device)
def __init__(self,
text_size: int,
embedding_size: int,
model_name: str,
optimizer: str = "Adam",
loss: str = "binary_crossentropy",
lear_rate: float = 0.01,
lear_rate_decay: float = 0.,
last_layer_activation="sigmoid",
confident_threshold: float = 0.5,
**kwargs):
"""
Initialize and train vocabularies, initializes embedder, tokenizer, and then initialize model using parameters
from opt dictionary (from config), if model is being initialized from saved.
"""
super().__init__(
text_size=text_size,
embedding_size=embedding_size,
model_name=model_name,
optimizer=optimizer,
loss=loss,
lear_rate=lear_rate,
lear_rate_decay=lear_rate_decay,
last_layer_activation=last_layer_activation,
confident_threshold=confident_threshold,
**kwargs) # self.opt = copy(kwargs) initialized in here
self.classes = list(np.sort(np.array(list(self.opt.get('classes')))))
self.opt['classes'] = self.classes
self.n_classes = len(self.classes)
if self.n_classes == 0:
ConfigError("Please, provide vocabulary with considered intents.")
self.opt['embedding_size'] = embedding_size
# Parameters required to init model
params = {
"model_name": self.opt.get('model_name'),
"optimizer_name": self.opt.get('optimizer'),
"loss_name": self.opt.get('loss'),
"lear_rate": self.opt.get('lear_rate'),
"lear_rate_decay": self.opt.get('lear_rate_decay')
}
self.model = self.load(**params)
self._change_not_fixed_params(
text_size=text_size,
embedding_size=embedding_size,
model_name=model_name,
optimizer=optimizer,
loss=loss,
lear_rate=lear_rate,
lear_rate_decay=lear_rate_decay,
last_layer_activation=last_layer_activation,
confident_threshold=confident_threshold,
**kwargs)
print("Model was successfully initialized!\nModel summary:\n{}".format(
self.model.summary()))
def decorate(model_cls: Type, reg_name: str = None) -> Type:
model_name = reg_name or short_name(model_cls)
global REGISTRY
if model_name in REGISTRY:
raise ConfigError(
'{} name is already registered'.format(model_name))
REGISTRY[model_name] = model_cls
return model_cls
def read(self,
data_path: Union[Path, str],
db_url: Optional[str] = None,
*args,
**kwargs) -> None:
"""Build a SQLite database from provided files, download SQLite database from a provided URL,
or do nothing.
Args:
data_path: a directory/file with texts to create a database from
db_url: path to a database url
kwargs:
save_path: a path where a database should be saved to, or path to a ready database
dataset_format: initial data format; should be selected from ['txt', 'wiki', 'json']
Returns:
None
"""
logger.info('Reading files...')
try:
save_path = expand_path(kwargs['save_path'])
except KeyError:
raise ConfigError(
f'\"save_path\" attribute should be set for {self.__class__.__name__}\
in the JSON config.')
if save_path.exists() and save_path.with_suffix(
f'{save_path.suffix}.done').exists():
return
try:
dataset_format = kwargs['dataset_format']
except KeyError:
raise ConfigError(
f'\"dataset_format\" attribute should be set for {self.__class__.__name__}\
in the JSON config.')
save_path.parent.mkdir(parents=True, exist_ok=True)
if db_url:
download_dir = save_path.parent
logger.info(
f'Downloading database from {db_url} to {download_dir}')
download(download_dir, db_url, force_download=False)
return
self._build_db(save_path, dataset_format, expand_path(data_path))
