def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
binary_feature_dim: int,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
label_smoothing: float = None,
ignore_span_metric: bool = False) -> None:
super(SemanticRoleLabeler, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SpanBasedF1Measure(vocab, tag_namespace="labels", ignore_classes=["V"])
self.encoder = encoder
# There are exactly 2 binary features for the verb predicate embedding.
self.binary_feature_embedding = Embedding(2, binary_feature_dim)
self.tag_projection_layer = TimeDistributed(Linear(self.encoder.get_output_dim(),
self.num_classes))
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
check_dimensions_match(text_field_embedder.get_output_dim() + binary_feature_dim,
encoder.get_input_dim(),
"text embedding dim + verb indicator embedding dim",
"encoder input dim")
initializer(self)
def __init__(self,
vocab: Vocabulary,
bert_model: Union[str, BertModel],
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH) -> None:
super().__init__(vocab, regularizer)
if isinstance(bert_model, str):
self.bert_model = BertModel.from_pretrained(bert_model)
else:
self.bert_model = bert_model
self.num_classes = self.vocab.get_vocab_size("labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.tag_projection_layer = Linear(self.bert_model.config.hidden_size,
self.num_classes)
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
initializer(self)
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
question_generator: QuestionGenerator,
stacked_encoder: Seq2SeqEncoder = None,
predicate_feature_dim: int = 100,
dim_hidden: int = 100,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None):
super(QuestionPredictor, self).__init__(vocab, regularizer)
self.dim_hidden = dim_hidden
self.text_field_embedder = text_field_embedder
self.predicate_feature_embedding = Embedding(2, predicate_feature_dim)
self.embedding_dropout = Dropout(p=embedding_dropout)
self.stacked_encoder = stacked_encoder
self.span_extractor = EndpointSpanExtractor(self.stacked_encoder.get_output_dim(), combination="x,y")
self.question_generator = question_generator
self.slot_labels = question_generator.get_slot_labels()
self.question_metric = QuestionPredictionMetric(vocab, question_generator.get_slot_labels())
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
tag_representation_dim: int,
arc_representation_dim: int,
tag_feedforward: FeedForward = None,
arc_feedforward: FeedForward = None,
dropout: float = 0.5,
input_dropout: float = 0.5,
head_tag_temperature: Optional[float] = None,
head_temperature: Optional[float] = None,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(Supertagger, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.encoder = encoder
encoder_dim = encoder.get_output_dim()
self.head_arc_feedforward = \
arc_feedforward or FeedForward(encoder_dim, 1,
arc_representation_dim,
Activation.by_name("elu")(),
dropout=dropout)
self.child_arc_feedforward = copy.deepcopy(self.head_arc_feedforward)
self.arc_attention = BilinearMatrixAttention(arc_representation_dim,
arc_representation_dim,
use_input_biases=True)
num_labels = self.vocab.get_vocab_size("head_tags")
self.head_tag_feedforward = \
tag_feedforward or FeedForward(encoder_dim, 1,
tag_representation_dim,
Activation.by_name("elu")(),
dropout=dropout)
self.child_tag_feedforward = copy.deepcopy(self.head_tag_feedforward)
self.tag_bilinear = BilinearWithBias(tag_representation_dim,
tag_representation_dim,
num_labels)
self._head_sentinel = torch.nn.Parameter(torch.randn([1, 1, encoder.get_output_dim()]))
representation_dim = text_field_embedder.get_output_dim()
check_dimensions_match(representation_dim, encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
check_dimensions_match(tag_representation_dim, self.head_tag_feedforward.get_output_dim(),
"tag representation dim", "tag feedforward output dim")
check_dimensions_match(arc_representation_dim, self.head_arc_feedforward.get_output_dim(),
"arc representation dim", "arc feedforward output dim")
self._input_dropout = Dropout(input_dropout)
self._attachment_scores = CategoricalAccuracy()
self._tagging_accuracy = CategoricalAccuracy()
self.head_tag_temperature = head_tag_temperature
self.head_temperature = head_temperature
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
edge_model: graph_dependency_parser.components.edge_models.
EdgeModel,
loss_function: graph_dependency_parser.components.losses.EdgeLoss,
pos_tag_embedding: Embedding = None,
use_mst_decoding_for_validation: bool = True,
dropout: float = 0.0,
input_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
validation_evaluator: Optional[ValidationEvaluator] = None
) -> None:
super(GraphDependencyParser, self).__init__(vocab, regularizer)
self.validation_evaluator = validation_evaluator
self.text_field_embedder = text_field_embedder
self.encoder = encoder
self._pos_tag_embedding = pos_tag_embedding or None
self._dropout = InputVariationalDropout(dropout)
self._input_dropout = Dropout(input_dropout)
self._head_sentinel = torch.nn.Parameter(
torch.randn([1, 1, encoder.get_output_dim()]))
representation_dim = text_field_embedder.get_output_dim()
if pos_tag_embedding is not None:
representation_dim += pos_tag_embedding.get_output_dim()
check_dimensions_match(representation_dim, encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
check_dimensions_match(encoder.get_output_dim(),
edge_model.encoder_dim(), "encoder output dim",
"input dim edge model")
self.use_mst_decoding_for_validation = use_mst_decoding_for_validation
tags = self.vocab.get_token_to_index_vocabulary("pos")
punctuation_tag_indices = {
tag: index
for tag, index in tags.items() if tag in POS_TO_IGNORE
}
self._pos_to_ignore = set(punctuation_tag_indices.values())
logger.info(
f"Found POS tags corresponding to the following punctuation : {punctuation_tag_indices}. "
"Ignoring words with these POS tags for evaluation.")
self._attachment_scores = AttachmentScores()
initializer(self)
self.edge_model = edge_model
self.loss_function = loss_function
#Being able to detect what state we are in, probably not the best idea.
self.current_epoch = 1
self.pass_over_data_just_started = True
def __init__(self,
options_file ,
weight_file ,
num_output_representations ,
requires_grad = False,
do_layer_norm = False,
dropout = 0.5,
vocab_to_cache = None,
module = None) :
super(Elmo, self).__init__()
logging.info(u"Initializing ELMo")
if module is not None:
if options_file is not None or weight_file is not None:
raise ConfigurationError(
u"Don't provide options_file or weight_file with module")
self._elmo_lstm = module
else:
self._elmo_lstm = _ElmoBiLm(options_file,
weight_file,
requires_grad=requires_grad,
vocab_to_cache=vocab_to_cache)
self._has_cached_vocab = vocab_to_cache is not None
self._dropout = Dropout(p=dropout)
self._scalar_mixes = []
for k in range(num_output_representations):
scalar_mix = ScalarMix(self._elmo_lstm.num_layers, do_layer_norm=do_layer_norm)
self.add_module(u'scalar_mix_{}'.format(k), scalar_mix)
self._scalar_mixes.append(scalar_mix)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
stacked_encoder: Seq2SeqEncoder,
predicate_feature_dim: int,
dim_hidden: int = 100,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None):
super(SpanDetector, self).__init__(vocab, regularizer)
self.dim_hidden = dim_hidden
self.text_field_embedder = text_field_embedder
self.predicate_feature_embedding = Embedding(2, predicate_feature_dim)
self.embedding_dropout = Dropout(p=embedding_dropout)
self.threshold_metric = ThresholdMetric()
self.stacked_encoder = stacked_encoder
self.span_hidden = SpanRepAssembly(
self.stacked_encoder.get_output_dim(),
self.stacked_encoder.get_output_dim(), self.dim_hidden)
self.pred = TimeDistributed(Linear(self.dim_hidden, 1))
def __init__(self, serialization_dir, cuda_device=0) -> None:
super(DependencyParsingEmbedding2, self).__init__()
from allennlp.models.archival import load_archive
self.serialization_dir = serialization_dir
self.parameter_filename = os.path.join(serialization_dir,
"config.json")
self.weights_filename = os.path.join(serialization_dir, "weights.th")
self.cuda_device = cuda_device
self.config = Params.from_file(self.parameter_filename)
self.archive = load_archive(self.serialization_dir)
self.model = self.archive.model
self.model.eval()
self.dataset_reader_params = self.config["dataset_reader"]
self.dataset_reader = DatasetReader.from_params(
self.dataset_reader_params)
self.tokenizer = SpacyWordSplitter(language='en_core_web_sm',
pos_tags=True,
wst=True)
num_output_representations = 1
do_layer_norm = False
num_layers = 3
self._keep_sentence_boundaries = False
self._dropout = Dropout(p=0.5)
self._scalar_mixes: Any = []
for k in range(num_output_representations):
scalar_mix = ScalarMix(num_layers, do_layer_norm=do_layer_norm)
self.add_module('scalar_mix_{}'.format(k), scalar_mix)
self._scalar_mixes.append(scalar_mix)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
stacked_encoder: Seq2SeqEncoder = None,
predicate_feature_dim: int = 0,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None):
super(SentenceEncoder, self).__init__()
self._text_field_embedder = text_field_embedder
self._stacked_encoder = stacked_encoder
self._predicate_feature_dim = predicate_feature_dim
self._embedding_dropout = Dropout(p=embedding_dropout)
if self._predicate_feature_dim > 0:
self._predicate_feature_embedding = Embedding(
2, predicate_feature_dim)
if self._stacked_encoder is not None:
embedding_dim_with_predicate_feature = self._text_field_embedder.get_output_dim(
) + self._predicate_feature_dim
if embedding_dim_with_predicate_feature != self._stacked_encoder.get_input_dim(
):
raise ConfigurationError(
("Input dimension of sentence encoder (%s) must be " % self._stacked_encoder.get_input_dim()) + \
("the sum of predicate feature dim and text embedding dim (%s)." % (embedding_dim_with_predicate_feature)))
self._metric = BinaryF1()
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
# binary_feature_dim: int,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(LstmSwag, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.encoder = encoder
self.embedding_dropout = Dropout(p=embedding_dropout)
self.output_prediction = Linear(self.encoder.get_output_dim(),
1,
bias=False)
check_dimensions_match(text_field_embedder.get_output_dim(),
encoder.get_input_dim(), "text embedding dim",
"eq encoder input dim")
self._accuracy = CategoricalAccuracy()
self._loss = torch.nn.CrossEntropyLoss()
initializer(self)
def __init__(self,
input_size,
output_size,
convs_configs,
dropout=.5,
normalize=True):
super(MultiCNN, self).__init__()
input_size = input_size
concat_cnn_output_size = 0
output_size = output_size
self._normalize = normalize
self.cnn_modules = nn.ModuleList()
for configs in convs_configs:
channel_size = configs['channel_size']
kernel_size = configs['kernel_size']
padding = configs['padding']
concat_cnn_output_size += channel_size
module = Conv1d(input_size,
channel_size,
kernel_size=kernel_size,
padding=padding)
self.cnn_modules.append(module)
self.batch_normlize = BatchNorm1d(concat_cnn_output_size)
self.dropout = Dropout(dropout)
self.output_linear = Linear(concat_cnn_output_size, output_size)
def __init__(self,
options_file: str,
weight_file: str,
num_output_representations: int,
requires_grad: bool = False,
do_layer_norm: bool = False,
dropout: float = 0.5,
module: torch.nn.Module = None) -> None:
super(Elmo, self).__init__()
logging.info("Initializing ELMo")
if module is not None:
if options_file is not None or weight_file is not None:
raise ConfigurationError(
"Don't provide options_file or weight_file with module")
self._elmo_lstm = module
else:
self._elmo_lstm = _ElmoBiLm(options_file,
weight_file,
requires_grad=requires_grad)
self._dropout = Dropout(p=dropout)
##################################################
# Eq.1 in the paper: gamma * sum(s_k * tensor_k) #
##################################################
self._scalar_mixes: Any = []
for k in range(num_output_representations):
# gamma * sum(s_k * tensor_k)
scalar_mix = ScalarMix(self._elmo_lstm.num_layers,
do_layer_norm=do_layer_norm)
self.add_module('scalar_mix_{}'.format(k), scalar_mix)
self._scalar_mixes.append(scalar_mix)
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
dropout: float = 0.0,
input_dropout: float = 0.0,
label_namespace: str = "pos",
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(PosTaggerMonolingual, self).__init__(vocab, regularizer)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size(label_namespace)
self.encoder = encoder
self._dropout = InputVariationalDropout(dropout)
self._input_dropout = Dropout(input_dropout)
self.tag_projection_layer = TimeDistributed(Linear(self.encoder.get_output_dim(),
self.num_classes))
check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3)
}
initializer(self)
def __init__(self,
options_file: str,
weight_file: str,
num_output_representations: int,
requires_grad: bool = False,
do_layer_norm: bool = False,
dropout: float = 0.5,
vocab_to_cache: List[str] = None,
keep_sentence_boundaries: bool = True,
scalar_mix_parameters: List[float] = None,
module: torch.nn.Module = None,
combine_method="weighted-sum",
random_init=False) -> None:
super(Elmo, self).__init__()
logger.info("Initializing ELMo")
self._elmo_lstm = _LatticeElmoBiLm(options_file,
weight_file,
requires_grad=requires_grad,
vocab_to_cache=vocab_to_cache,
combine_method=combine_method,
random_init=random_init)
self._has_cached_vocab = vocab_to_cache is not None
self._keep_sentence_boundaries = keep_sentence_boundaries
self._dropout = Dropout(p=dropout)
self._scalar_mixes: Any = []
for k in range(num_output_representations):
scalar_mix = ScalarMix(
self._elmo_lstm.num_layers,
do_layer_norm=do_layer_norm,
initial_scalar_parameters=scalar_mix_parameters,
trainable=scalar_mix_parameters is None,
)
self.add_module("scalar_mix_{}".format(k), scalar_mix)
self._scalar_mixes.append(scalar_mix)
def __init__(
self,
options_file: str,
weight_file: str,
num_output_representations: int,
requires_grad: bool = False,
do_layer_norm: bool = False,
dropout: float = 0.5,
vocab_to_cache: List[str] = None,
module: torch.nn.Module = None,
) -> None:
super(Elmo, self).__init__()
logging.info("Initializing ELMo")
if module is not None:
if options_file is not None or weight_file is not None:
raise ValueError("Don't provide options_file or weight_file with module")
self._elmo_lstm = module
else:
self._elmo_lstm = _ElmoBiLm(
options_file,
weight_file,
requires_grad=requires_grad,
vocab_to_cache=vocab_to_cache,
)
self._has_cached_vocab = vocab_to_cache is not None
self._dropout = Dropout(p=dropout)
self._scalar_mixes: Any = []
for k in range(num_output_representations):
scalar_mix = ScalarMix(self._elmo_lstm.num_layers, do_layer_norm=do_layer_norm)
self.add_module("scalar_mix_{}".format(k), scalar_mix)
self._scalar_mixes.append(scalar_mix)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
stacked_encoder: Seq2SeqEncoder,
binary_feature_dim: int,
initializer: InitializerApplicator,
embedding_dropout: float = 0.0) -> None:
super(SemanticRoleLabeler, self).__init__(vocab)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SpanBasedF1Measure(vocab,
tag_namespace="labels",
ignore_classes=["V"])
self.stacked_encoder = stacked_encoder
# There are exactly 2 binary features for the verb predicate embedding.
self.binary_feature_embedding = Embedding(2, binary_feature_dim)
self.tag_projection_layer = TimeDistributed(
Linear(self.stacked_encoder.get_output_dim(), self.num_classes))
self.embedding_dropout = Dropout(p=embedding_dropout)
initializer(self)
if text_field_embedder.get_output_dim(
) + binary_feature_dim != stacked_encoder.get_input_dim():
raise ConfigurationError(
"The SRL Model uses a binary verb indicator feature, meaning "
"the input dimension of the stacked_encoder must be equal to "
"the output dimension of the text_field_embedder + 1.")
def __init__(self,
vocab: Vocabulary,
embedder: TextFieldEmbedder,
encoder: Seq2VecEncoder,
feedforward: Optional[FeedForward] = None,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
dropout: float = 0.0,
label_name: str = 'target-sentiment-labels') -> None:
super().__init__(vocab, regularizer)
'''
:param vocab: A Vocabulary, required in order to compute sizes
for input/output projections.
:param embedder: Used to embed the text.
:param encoder: Encodes the sentence/text. E.g. LSTM
:param feedforward: An optional feed forward layer to apply after the
encoder
:param initializer: Used to initialize the model parameters.
:param regularizer: If provided, will be used to calculate the
regularization penalty during training.
:param dropout: To apply dropout after each layer apart from the last
layer. All dropout that is applied to timebased data
will be `variational dropout`_ all else will be
standard dropout.
:param label_name: Name of the label name space.
This is based on the LSTM model by
`Tang et al. 2016 <https://www.aclweb.org/anthology/C16-1311.pdf>`_
'''
self.label_name = label_name
self.embedder = embedder
self.encoder = encoder
self.num_classes = self.vocab.get_vocab_size(self.label_name)
self.feedforward = feedforward
if feedforward is not None:
output_dim = self.feedforward.get_output_dim()
else:
output_dim = self.encoder.get_output_dim()
self.label_projection = Linear(output_dim, self.num_classes)
self.metrics = {"accuracy": CategoricalAccuracy()}
self.f1_metrics = {}
# F1 Scores
label_index_name = self.vocab.get_index_to_token_vocabulary(
self.label_name)
for label_index, _label_name in label_index_name.items():
_label_name = f'F1_{_label_name.capitalize()}'
self.f1_metrics[_label_name] = F1Measure(label_index)
self._variational_dropout = InputVariationalDropout(dropout)
self._naive_dropout = Dropout(dropout)
check_dimensions_match(embedder.get_output_dim(),
encoder.get_input_dim(), 'Embedding', 'Encoder')
if self.feedforward is not None:
check_dimensions_match(encoder.get_output_dim(),
feedforward.get_input_dim(), 'Encoder',
'FeedForward')
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
tag_representation_dim: int,
arc_representation_dim: int,
pos_tag_embedding: Embedding = None,
use_mst_decoding_for_validation: bool = True,
dropout: float = 0.0,
input_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(BiaffineDependencyParser, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.encoder = encoder
encoder_dim = encoder.get_output_dim()
self.head_arc_projection = torch.nn.Linear(encoder_dim,
arc_representation_dim)
self.child_arc_projection = torch.nn.Linear(encoder_dim,
arc_representation_dim)
self.arc_attention = BilinearMatrixAttention(arc_representation_dim,
arc_representation_dim,
use_input_biases=True)
num_labels = self.vocab.get_vocab_size("head_tags")
self.head_tag_projection = torch.nn.Linear(encoder_dim,
tag_representation_dim)
self.child_tag_projection = torch.nn.Linear(encoder_dim,
tag_representation_dim)
self.tag_bilinear = torch.nn.modules.Bilinear(tag_representation_dim,
tag_representation_dim,
num_labels)
self._pos_tag_embedding = pos_tag_embedding or None
self._dropout = InputVariationalDropout(dropout)
self._input_dropout = Dropout(input_dropout)
representation_dim = text_field_embedder.get_output_dim()
if pos_tag_embedding is not None:
representation_dim += pos_tag_embedding.get_output_dim()
check_dimensions_match(representation_dim, encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
self.use_mst_decoding_for_validation = use_mst_decoding_for_validation
tags = self.vocab.get_token_to_index_vocabulary("pos")
punctuation_tag_indices = {
tag: index
for tag, index in tags.items() if tag in POS_TO_IGNORE
}
self._pos_to_ignore = set(punctuation_tag_indices.values())
logger.info(
f"Found POS tags correspoding to the following punctuation : {punctuation_tag_indices}. "
"Ignoring words with these POS tags for evaluation.")
self._attachment_scores = AttachmentScores()
initializer(self)
def __init__(self,
d_model,
nhead,
dim_feedforward=2048,
dropout=0.1,
activation="relu"):
super(TransformerEncoderLayer, self).__init__()
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout)
# Implementation of Feedforward model
self.linear1 = Linear(d_model, dim_feedforward)
self.dropout = Dropout(dropout)
self.linear2 = Linear(dim_feedforward, d_model)
self.norm1 = LayerNorm(d_model)
self.norm2 = LayerNorm(d_model)
self.dropout1 = Dropout(dropout)
self.dropout2 = Dropout(dropout)
self.activation = _get_activation_fn(activation)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
binary_feature_dim: int,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
label_smoothing: float = None,
label_namespace: str = "labels",
ignore_span_metric: bool = False,
label_encoding: Optional[str] = 'BIO',
include_start_end_transitions: bool = True,
constrain_crf_decoding: bool = True) -> None:
super(OieLabelerCRF, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SpanBasedF1Measure(vocab,
tag_namespace="labels",
ignore_classes=["V"])
self.label_namespace = label_namespace
self.encoder = encoder
# There are exactly 2 binary features for the verb predicate embedding.
self.binary_feature_embedding = Embedding(2, binary_feature_dim)
self.tag_projection_layer = TimeDistributed(
Linear(self.encoder.get_output_dim(), self.num_classes))
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
self.include_start_end_transitions = include_start_end_transitions
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
if constrain_crf_decoding:
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
print(labels)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.crf = ConditionalRandomField(
self.num_classes,
constraints,
include_start_end_transitions=include_start_end_transitions)
check_dimensions_match(
text_field_embedder.get_output_dim() + binary_feature_dim,
encoder.get_input_dim(),
"text embedding dim + verb indicator embedding dim",
"encoder input dim")
initializer(self)
def __init__(self,
vocab: Vocabulary,
source_embedder: TextFieldEmbedder,
sentence_encoder: Seq2VecEncoder,
sentence_accumulator: Seq2SeqEncoder,
use_salience: bool,
use_pos_embedding: bool,
use_output_bias: bool,
use_novelty: bool,
dropout: float = 0.3,
pos_embedding_num: int = 50,
pos_embedding_size: int = 128) -> None:
super(SummaRuNNer, self).__init__(vocab)
self._source_embedder = source_embedder
self._sentence_encoder = sentence_encoder
self._se_output_dim = self._sentence_encoder.get_output_dim()
self._sentence_accumulator = sentence_accumulator
self._h_sentence_dim = self._sentence_accumulator.get_output_dim()
self._dropout_layer = Dropout(dropout)
self._content_projection_layer = Linear(self._h_sentence_dim, 1)
self._use_salience = use_salience
if use_salience:
self._document_linear_layer = Linear(self._h_sentence_dim,
self._h_sentence_dim,
bias=True)
self._salience_linear_layer = Linear(self._h_sentence_dim,
self._h_sentence_dim,
bias=False)
self._use_pos_embedding = use_pos_embedding
if use_pos_embedding:
self._pos_embedding_num = pos_embedding_num
self._pos_embedding_size = pos_embedding_size
self._pos_embedding_layer = Embedding(pos_embedding_num,
pos_embedding_size)
self._pos_projection_layer = Linear(pos_embedding_size, 1)
self._use_output_bias = use_output_bias
if use_output_bias:
self._output_bias = Parameter(torch.zeros(1).uniform_(-0.1, 0.1),
requires_grad=True)
self._use_novelty = use_novelty
if use_novelty:
self._novelty_linear_layer = Linear(self._h_sentence_dim,
self._h_sentence_dim,
bias=False)
def __init__(
self,
vocab: Vocabulary,
rnn: Seq2SeqEncoder,
latent_dim: int,
target_embedder: TextFieldEmbedder,
target_namespace: str = 'tokens',
dropout_p: float = 0.5,
) -> None:
super().__init__(vocab)
self._target_embedder = target_embedder
self._target_namespace = target_namespace
self.rnn = rnn
self.dec_num_layers = self.rnn._module.num_layers # noqa: WPS437
self.dec_hidden = self.rnn._module.hidden_size # noqa: WPS437
self._latent_to_dec_hidden = Linear(latent_dim, self.dec_hidden)
self._dec_linear = Linear(self.rnn.get_output_dim(),
self.vocab.get_vocab_size())
self._emb_dropout = Dropout(dropout_p)
self._dec_dropout = Dropout(dropout_p)
def __init__(self, rnn_size, dropout=DROPOUT, max_len=5000):
super(PositionalEncoding, self).__init__()
self.dropout = Dropout(p=dropout)
# Compute the positional encodings once in log space
pe = torch.zeros(max_len, rnn_size)
position = torch.arange(0.0, max_len).unsqueeze(1)
div_term = torch.exp(
torch.arange(0.0, rnn_size, 2) * -(math.log(10000.0) / rnn_size))
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0)
self.register_buffer('pe', pe)
def __init__(self,
vocab: Vocabulary,
target_embedder: TextFieldEmbedder,
rnn: Seq2SeqEncoder,
latent_dim: int,
dropout_p: float = 0.5) -> None:
super(VariationalDecoder, self).__init__()
self.vocab = vocab
self._target_embedder = target_embedder
self.rnn = rnn
self.dec_num_layers = self.rnn._module.num_layers # pylint: disable=protected-access
self.dec_hidden = self.rnn._module.hidden_size # pylint: disable=protected-access
self._latent_to_dec_hidden = Linear(latent_dim, self.dec_hidden)
self._dec_linear = Linear(self.rnn.get_output_dim(),
self.vocab.get_vocab_size())
self._emb_dropout = Dropout(dropout_p)
self._dec_dropout = Dropout(dropout_p)
self._start_index = self.vocab.get_token_index(START_SYMBOL)
self._end_index = self.vocab.get_token_index(END_SYMBOL)
def __init__(self,
vocab: Vocabulary,
source_embedder: TextFieldEmbedder,
sentence_encoder: Seq2VecEncoder,
sentence_accumulator: Seq2SeqEncoder,
use_salience: bool,
use_pos_embedding: bool,
use_output_bias: bool,
use_novelty: bool,
dropout: float = 0.3,
pos_embedding_num: int = 50,
pos_embedding_size: int = 128) -> None:
super().__init__(vocab)
self._source_embedder = source_embedder
self._sentence_encoder = sentence_encoder
self._se_output_dim = self._sentence_encoder.get_output_dim()
self._sentence_accumulator = sentence_accumulator
self._h_sentence_dim = self._sentence_accumulator.get_output_dim()
self._dropout_layer = Dropout(dropout)
self._content_projection_layer = Linear(self._h_sentence_dim, 1)
# options to train the system on
self._use_pos_embedding = use_pos_embedding
self._use_salience = use_salience
self._use_novelty = use_novelty
if use_salience:
self._document_linear_layer = Linear(self._h_sentence_dim,
self._h_sentence_dim,
bias=True)
self._salience_linear_layer = Linear(self._h_sentence_dim,
self._h_sentence_dim,
bias=False)
if use_pos_embedding:
self._pos_embedding_num = pos_embedding_num
self._pos_embedding_size = pos_embedding_size
self._pos_embedding_layer = Embedding(pos_embedding_num,
pos_embedding_size)
self._pos_projection_layer = Linear(pos_embedding_size, 1)
self._use_novelty = use_novelty
if use_novelty:
self._novelty_linear_layer = Linear(self._h_sentence_dim,
self._h_sentence_dim,
bias=False)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: MTLWeightSharer,
tasks: List[AMTask],
pos_tag_embedding: Embedding = None,
lemma_embedding: Embedding = None,
ne_embedding: Embedding = None,
input_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
tok2vec: Optional[TokenToVec] = None) -> None:
super(GraphDependencyParser, self).__init__(vocab, regularizer)
self.text_field_embedder = text_field_embedder
self.encoder = encoder
self.tok2vec = tok2vec
self._pos_tag_embedding = pos_tag_embedding or None
self._lemma_embedding = lemma_embedding
self._ne_embedding = ne_embedding
self._input_dropout = Dropout(input_dropout)
self._head_sentinel = torch.nn.Parameter(
torch.randn([1, 1, encoder.get_output_dim()]))
representation_dim = text_field_embedder.get_output_dim()
if pos_tag_embedding is not None:
representation_dim += pos_tag_embedding.get_output_dim()
if self._lemma_embedding is not None:
representation_dim += lemma_embedding.get_output_dim()
if self._ne_embedding is not None:
representation_dim += ne_embedding.get_output_dim()
assert len(tasks) > 0, "List of tasks must not be empty"
self.tasks: Dict[str, AMTask] = {t.name: t for t in tasks}
if self.tok2vec:
representation_dim += self.tok2vec.get_output_dim()
check_dimensions_match(representation_dim, encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
for t in tasks:
t.check_all_dimensions_match(encoder.get_output_dim())
for formalism, task in sorted(self.tasks.items(),
key=lambda nt: nt[0]):
#sort by name of formalism for consistent ordering
self.add_module(formalism, task)
initializer(self)
def __init__(self,
options_file: str,
weight_file: str,
num_output_representations: int,
do_layer_norm: bool = False,
dropout: float = 0.5) -> None:
super(Elmo, self).__init__()
self._elmo_lstm = _ElmoBiLm(options_file, weight_file)
self._dropout = Dropout(p=dropout)
self._scalar_mixes: Any = []
for k in range(num_output_representations):
scalar_mix = ScalarMix(self._elmo_lstm.num_layers, do_layer_norm=do_layer_norm)
self.add_module('scalar_mix_{}'.format(k), scalar_mix)
self._scalar_mixes.append(scalar_mix)
def __init__(self, in_size: int, out_size: int, dropout: float) -> None:
"""
:param in_size: dimensionality of input vectors
:param out_size: dimensionality of hidden states and output vectors
:param dropout: dropout ratio
"""
super().__init__()
self.in_size = in_size
self.state_size = out_size
if dropout == 0.0:
self._dropout = lambda x: x
else:
self._dropout = Dropout(dropout)
self._topdown_lstm = ChildSumTreeLSTM(in_size, out_size)
self._bottomup_lstm = ChildSumTreeLSTM(in_size, out_size)
def __init__(self,
vocab: Vocabulary,
bert_model: Union[str, BertModel],
pretrained_conditional_semanticbert: str,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
label_smoothing: float = None,
ignore_span_metric: bool = False) -> None:
super(SrlBert, self).__init__(vocab, regularizer)
if isinstance(bert_model, str):
self.bert_model = BertModel.from_pretrained(bert_model)
else:
self.bert_model = bert_model
# load pre-trained conditional semanticbert
self.conditional_semanticbert = ConditionalSemanticBert.from_pretrained(
"bert-base-uncased")
print('load a pre-trained model from ' +
pretrained_conditional_semanticbert)
pretrained_state_dict = torch.load(pretrained_conditional_semanticbert)
model_state_dict = self.conditional_semanticbert.state_dict()
print('pretrained_state_dict', pretrained_state_dict.keys())
print('model_state_dict', model_state_dict.keys())
pretrained_state = {
k: v
for k, v in pretrained_state_dict.items() if k in model_state_dict
and v.size() == model_state_dict[k].size()
}
model_state_dict.update(pretrained_state)
print('updated_state_dict', model_state_dict.keys())
self.conditional_semanticbert.load_state_dict(model_state_dict)
self.conditional_semanticbert.to('cuda')
self.num_classes = self.vocab.get_vocab_size("labels")
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SpanBasedF1Measure(vocab,
tag_namespace="labels",
ignore_classes=["V"])
self.tag_projection_layer = Linear(
self.bert_model.config.hidden_size * 2, self.num_classes)
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
dropout: float = 0.0,
input_dropout: float = 0.0,
label_namespace: str = "pos",
treebank_embedding: Embedding = None,
use_treebank_embedding: bool = True,
langs_for_early_stop: List[str] = None,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(PosTaggerTbemb, self).__init__(vocab, regularizer)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size(label_namespace)
self.encoder = encoder
self._dropout = InputVariationalDropout(dropout)
self._input_dropout = Dropout(input_dropout)
self._langs_for_early_stop = langs_for_early_stop or []
self._treebank_embedding = treebank_embedding or None
self._use_treebank_embedding = use_treebank_embedding
self._lang_accuracy_scores: Dict[
str, CategoricalAccuracy] = defaultdict(CategoricalAccuracy)
self.tag_projection_layer = TimeDistributed(
Linear(self.encoder.get_output_dim(), self.num_classes))
representation_dim = text_field_embedder.get_output_dim()
if treebank_embedding is not None:
representation_dim += treebank_embedding.get_output_dim()
check_dimensions_match(representation_dim, encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
if self._use_treebank_embedding:
tbids = self.vocab.get_token_to_index_vocabulary("tbids")
tbid_indices = {tb: index for tb, index in tbids.items()}
self._tbids = set(tbid_indices.values())
logger.info(
f"Found TBIDs corresponding to the following treebanks : {tbid_indices}. "
"Embedding these as additional features.")
initializer(self)
