def __init__(self, options, train_graphs=None, restore_model_and_saveable=None, train_extra=None):
random.seed(1)
self.decoder = graph_decoders[options.decoder](options)
self.test_decoder = graph_decoders[options.test_decoder](options) \
if options.test_decoder is not None \
else self.decoder
self.cost_augment = cost_augmentors[options.cost_augment](options)
self.options = options
if "func" in options:
del options.func
self.labelsFlag = options.labelsFlag
if restore_model_and_saveable:
self.model, self.network = restore_model_and_saveable
else:
self.model = dn.Model()
self.trainer = nn.trainers[options.trainer](self.model)
if train_extra is not None:
statistics = Statistics.from_sentences(train_graphs + train_extra)
else:
statistics = Statistics.from_sentences(train_graphs)
self.network = EdgeEvaluationNetwork(self.model, statistics, options)
def __init__(self, model, statistics, options):
super(EdgeDoubleEvaluationNetwork, self).__init__(model)
self.share_embedding = options.share_embedding
self.share_lstm = options.share_lstm
self.graph_network = EdgeEvaluationNetwork(model, statistics[0],
options)
self.tree_network = EdgeEvaluationNetwork(model, statistics[1],
options)
self.set_share_parameter(model)
def add_parser_arguments(cls, arg_parser):
super(MaxSubTreeParser, cls).add_parser_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--optimizer", type=str, dest="optimizer", default="adam", choices=nn.trainers.keys())
group.add_argument("--decoder", type=str, dest="decoder", default="eisner", choices=decoders)
group.add_argument("--cost-augment", action="store_true", dest="cost_augment", default=True)
group.add_argument("--batch-size", type=int, dest="batch_size", default=cls.default_batch_size)
group.add_argument("--model-format", dest="model_format", choices=nn.model_formats, default="pickle")
EdgeEvaluationNetwork.add_parser_arguments(arg_parser)
def __init__(self, options, train_sentences=None):
self.model = dn.Model()
self.statistics = Statistics.from_sentences(train_sentences)
self.container = nn.Container(self.model)
self.network = EdgeEvaluationNetwork(self.container, self.statistics, options)
self.optimizer = nn.get_optimizer(self.model, options)
self.decoder = decoders[options.decoder]
self.label_decoder = label_decoders[options.label_decoder]
self.labelsFlag = options.labelsFlag
self.options = options
if "func" in options:
del options.func
def add_parser_arguments(cls, arg_parser):
""":type arg_parser: argparse.ArgumentParser"""
EdgeEvaluationNetwork.add_parser_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--not-share-embedding",
action="store_false",
dest="share_embedding",
default=True)
group.add_argument("--not-share-lstm",
action="store_false",
dest="share_lstm",
default=True)
def __init__(self,
options,
train_sentences=None,
restore_file=None,
statistics=None):
self.model = dn.Model()
random.seed(1)
self.trainer = dn.AdamTrainer(self.model)
self.activation = activations[options.activation]
# self.decoder = decoders[options.decoder]
self.labelsFlag = options.labelsFlag
self.costaugFlag = options.cost_augment
self.options = options
if "func" in options:
del options.func
if restore_file:
self.container, = dn.load(restore_file, self.model)
networks = list(self.container.components)
self.network = networks.pop(0)
self.statistics = statistics
self.has_emptys = len(statistics.emptys) > 0
if self.has_emptys:
self.network_for_emptys = networks.pop(0)
if self.options.use_2nd:
self.network3 = networks.pop(0)
if self.has_emptys:
self.network3_for_emptys_mid = networks.pop(0)
self.network3_for_emptys_out = networks.pop(0)
assert not networks
else:
self.container = nn.Container(self.model)
self.statistics = statistics = StatisticsWithEmpty.from_sentences(
train_sentences)
self.has_emptys = len(statistics.emptys) > 0
self.network = EdgeEvaluationNetwork(self.container, statistics,
options)
if self.has_emptys:
self.network_for_emptys = EdgeEvaluation(
self.container, options)
if options.use_2nd:
self.network3 = EdgeSiblingEvaluation(self.container, options)
if self.has_emptys:
self.network3_for_emptys_mid = EdgeSiblingEvaluation(
self.container, options)
self.network3_for_emptys_out = EdgeSiblingEvaluation(
self.container, options)
def add_parser_arguments(cls, arg_parser):
super(MaxSubTreeParser, cls).add_parser_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--decoder",
type=str,
dest="decoder",
default="eisner2nd",
choices=["eisner2nd"])
group.add_argument("--cost-augment",
action="store_true",
dest="cost_augment",
default=True)
EdgeEvaluationNetwork.add_parser_arguments(arg_parser)
def add_parser_arguments(cls, arg_parser):
""":type arg_parser: argparse.ArgumentParser"""
super(MaxSubGraphParser, cls).add_parser_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--trainer", type=str, dest="trainer", default="adam", choices=nn.trainers.keys())
group.add_argument("--cost-augment", type=str, dest="cost_augment", default="hamming", choices=cost_augmentors)
group.add_argument("--decoder", type=str, dest="decoder", default="arcfactor", choices=graph_decoders.keys())
group.add_argument("--predict-decoder", type=str, dest="test_decoder", default=None)
group.add_argument("--hamming-a", type=float, dest="hamming_a", default=0.4)
group.add_argument("--hamming-b", type=float, dest="hamming_b", default=0.6)
group.add_argument("--vine-arc-length", type=int, dest="vine_arc_length", default=20)
group.add_argument("--basic-costaug-decrease", type=int, dest="basic_costaug_decrease", default=1)
group.add_argument("--loose-value", type=float, dest="loose", default=-1)
group.add_argument("--delta", type=float, dest="delta", default=1)
EdgeEvaluationNetwork.add_parser_arguments(arg_parser)
def __init__(self, options, train_sentences=None, restore_file=None):
self.model = dn.Model()
if restore_file:
old_options, self.network = nn.model_load_helper(None, restore_file, self.model)
if options is not None:
old_options.__dict__.update(options.__dict__)
options = old_options
else:
statistics = Statistics.from_sentences(train_sentences)
self.network = EdgeEvaluationNetwork(self.model, statistics, options)
self.optimizer = nn.get_optimizer(self.model, options)
self.decoder = decoders[options.decoder]
self.labelsFlag = options.labelsFlag
self.options = options
if "func" in options:
del options.func
def add_parser_arguments(cls, arg_parser):
super(MaxSubTreeWithEmptyParser, cls).add_parser_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--optimizer",
type=str,
default="adam",
choices=nn.trainers.keys())
group.add_argument("--decoder",
type=str,
dest="decoder",
default="eisner2nd",
choices=["eisner2nd"])
group.add_argument("--disable-cost-augment",
action="store_false",
dest="cost_augment",
default=True)
group.add_argument("--enable-2nd",
action="store_true",
dest="use_2nd",
default=False)
EdgeEvaluationNetwork.add_parser_arguments(arg_parser)
def __init__(self,
options,
train_sentences=None,
restore_file=None,
statistics=None):
self.model = dn.Model()
random.seed(1)
self.optimizer = nn.get_optimizer(self.model, options)
self.activation = activations[options.activation]
# self.decoder = decoders[options.decoder]
self.labelsFlag = options.labelsFlag
self.costaugFlag = options.cost_augment
self.options = options
if "func" in options:
del options.func
self.container = nn.Container(self.model)
self.statistics = statistics = StatisticsWithEmpty.from_sentences(
train_sentences)
self.has_emptys = len(statistics.emptys) > 0
self.network = EdgeEvaluationNetwork(self.container, statistics,
options)
if self.has_emptys:
self.network_for_emptys = EdgeEvaluation(self.container, options)
self.label_network_for_emptys = ToEmptyLabelEvaluation(
self.container, self.statistics.emptys, options)
if options.use_2nd:
self.network3 = EdgeSiblingEvaluation(self.container, options)
if self.has_emptys:
self.network3_for_emptys_mid = EdgeSiblingEvaluation(
self.container, options)
self.network3_for_emptys_out = EdgeSiblingEvaluation(
self.container, options)
def __init__(self, options, train_sentences=None, restore_file=None):
self.model = dn.Model()
random.seed(1)
self.trainer = dn.AdamTrainer(self.model)
self.activation = activations[options.activation]
self.decoder = decoders[options.decoder]
self.labelsFlag = options.labelsFlag
self.costaugFlag = options.cost_augment
self.options = options
if "func" in options:
del options.func
if restore_file:
self.container, = dn.load(restore_file, self.model)
self.network, self.network3 = self.container.components
else:
self.container = nn.Container(self.model)
statistics = Statistics.from_sentences(train_sentences)
self.network = EdgeEvaluationNetwork(self.container, statistics,
options)
self.network3 = EdgeSiblingEvaluation(self.container, options)
class MaxSubGraphParser(GraphParserBase):
@classmethod
def add_parser_arguments(cls, arg_parser):
""":type arg_parser: argparse.ArgumentParser"""
super(MaxSubGraphParser, cls).add_parser_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--trainer", type=str, dest="trainer", default="adam", choices=nn.trainers.keys())
group.add_argument("--cost-augment", type=str, dest="cost_augment", default="hamming", choices=cost_augmentors)
group.add_argument("--decoder", type=str, dest="decoder", default="arcfactor", choices=graph_decoders.keys())
group.add_argument("--predict-decoder", type=str, dest="test_decoder", default=None)
group.add_argument("--hamming-a", type=float, dest="hamming_a", default=0.4)
group.add_argument("--hamming-b", type=float, dest="hamming_b", default=0.6)
group.add_argument("--vine-arc-length", type=int, dest="vine_arc_length", default=20)
group.add_argument("--basic-costaug-decrease", type=int, dest="basic_costaug_decrease", default=1)
group.add_argument("--loose-value", type=float, dest="loose", default=-1)
group.add_argument("--delta", type=float, dest="delta", default=1)
EdgeEvaluationNetwork.add_parser_arguments(arg_parser)
@classmethod
def add_common_arguments(cls, arg_parser):
super(MaxSubGraphParser, cls).add_common_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__ + " (common)")
group.add_argument("--batch-size", type=int, dest="batch_size", default=32)
def __init__(self, options, train_graphs=None, restore_model_and_saveable=None, train_extra=None):
random.seed(1)
self.decoder = graph_decoders[options.decoder](options)
self.test_decoder = graph_decoders[options.test_decoder](options) \
if options.test_decoder is not None \
else self.decoder
self.cost_augment = cost_augmentors[options.cost_augment](options)
self.options = options
if "func" in options:
del options.func
self.labelsFlag = options.labelsFlag
if restore_model_and_saveable:
self.model, self.network = restore_model_and_saveable
else:
self.model = dn.Model()
self.trainer = nn.trainers[options.trainer](self.model)
if train_extra is not None:
statistics = Statistics.from_sentences(train_graphs + train_extra)
else:
statistics = Statistics.from_sentences(train_graphs)
self.network = EdgeEvaluationNetwork(self.model, statistics, options)
def predict_session(self, sentence):
"""
step 1: yield all edge expressions
step 2: yield all label expressions
step 3: yield result graph
"""
lstm_output = self.network.get_lstm_output(sentence)
length = len(sentence)
raw_exprs = self.network.edge_eval.get_complete_raw_exprs(lstm_output)
yield raw_exprs
scores = self.network.edge_eval.raw_exprs_to_scores(raw_exprs, length)
output_graph = self.test_decoder(scores)
edges = []
for source_id in range(len(sentence)):
for target_id in range(len(sentence)):
if target_id == 0: # avoid edges pointed to root
continue
if output_graph[source_id][target_id]:
edges.append(graph_utils.Edge(source_id, "X", target_id))
if self.labelsFlag:
labeled_edges = []
labels_exprs = list(self.network.label_eval.get_label_scores(lstm_output, edges))
yield labels_exprs
for edge, r_scores_expr in zip(edges, labels_exprs):
r_scores = r_scores_expr.value()
label_index = max(((l, scr) for l, scr in enumerate(r_scores)), key=itemgetter(1))[0]
label = self.network.irels[label_index]
labeled_edges.append(graph_utils.Edge(edge.source, label, edge.target))
edges = labeled_edges
else:
yield []
result = sentence.replaced_edges(edges)
yield result
def predict(self, graphs):
self.network.sent_embedding.rnn.disable_dropout()
for sentence_idx, batch_idx, batch_sentences in split_to_batches(
graphs, self.options.batch_size):
sessions = [self.predict_session(sentence)
for sentence in batch_sentences]
all_exprs = [next(i) for i in sessions]
if all_exprs:
dn.forward(all_exprs)
all_labels_exprs = [j for i in sessions for j in next(i)]
if all_labels_exprs:
dn.forward(all_labels_exprs)
for i in sessions:
yield next(i)
dn.renew_cg()
def training_session(self, sentence, print_logger, loose_var=-1):
"""
step 1: yield all edge expressions
step 2: yield all label expressions
step 3: yield loss
"""
lstm_output = self.network.get_lstm_output(sentence)
length = len(sentence)
raw_exprs = self.network.edge_eval.get_complete_raw_exprs(lstm_output)
yield raw_exprs
scores = self.network.edge_eval.raw_exprs_to_scores(raw_exprs, length)
exprs = self.network.edge_eval.raw_exprs_to_exprs(raw_exprs, length)
self.cost_augment(scores, sentence)
output_graph = self.decoder(scores)
gold_graph = sentence.to_matrix()
label_loss = dn.scalarInput(0.0)
if self.labelsFlag:
edges = list(sentence.generate_edges())
labels_exprs = list(self.network.label_eval.get_label_scores(lstm_output, edges))
yield labels_exprs
for edge, r_scores_expr \
in zip(edges, labels_exprs):
head, label, modifier = edge
r_scores = r_scores_expr.value()
gold_label_index = self.network.rels[label]
wrong_label_index = max(((l, scr)
for l, scr in enumerate(r_scores)
if l != gold_label_index), key=itemgetter(1))[0]
# if loose_var is set, we could do something to loose the update of tagging
delta = self.options.delta if loose_var > 0 else 1
if r_scores[gold_label_index] < r_scores[wrong_label_index] + delta:
label_loss += r_scores_expr[wrong_label_index] - r_scores_expr[gold_label_index] + 1
else:
yield []
edge_loss = dn.scalarInput(0.0)
for source_id in range(len(sentence)):
for target_id in range(len(sentence)):
gold_exist = gold_graph[source_id][target_id]
output_exist = output_graph[source_id][target_id]
if gold_exist and output_exist:
print_logger.total_gold_edge += 1
print_logger.total_predict_edge += 1
print_logger.correct_predict_edge += 1
print_logger.recalled_gold_edge += 1
elif not gold_exist and not output_exist:
pass
elif gold_exist and not output_exist:
print_logger.total_gold_edge += 1
if loose_var > 0 and scores[source_id][target_id] > -loose_var: #-0.1:
pass
else:
edge_loss -= exprs[source_id][target_id]
elif not gold_exist and output_exist:
print_logger.total_predict_edge += 1
if loose_var > 0 and scores[source_id][target_id] < loose_var: #0.05:
pass
else:
edge_loss += exprs[source_id][target_id]
else:
raise SystemError()
loss_shift = self.cost_augment.get_loss_shift(output_graph, gold_graph)
loss = label_loss + edge_loss + loss_shift
yield loss
def train_gen(self, graphs, update=True, extra=None):
"""
:type graphs: list[graph_utils.Graph]
"""
self.logger = PrintLogger()
self.network.sent_embedding.rnn.set_dropout(self.options.lstm_dropout)
print_per = (100 // self.options.batch_size + 1) * self.options.batch_size
if extra is not None:
for sentence_idx, batch_idx, batch_sentences in split_to_batches(
extra, self.options.batch_size):
if sentence_idx % print_per == 0 and sentence_idx != 0:
self.logger.print(sentence_idx)
sessions = [self.training_session(sentence, self.logger, loose_var=self.options.loose)
for sentence in batch_sentences]
all_exprs = [next(i) for i in sessions]
if all_exprs:
dn.forward(all_exprs)
all_labels_exprs = [j for i in sessions for j in next(i)]
if all_labels_exprs:
dn.forward(all_labels_exprs)
loss = sum(next(i) for i in sessions) / len(sessions)
self.logger.total_loss_value += loss.value()
if update:
loss.backward()
self.trainer.update()
dn.renew_cg()
sessions.clear()
for sentence_idx, batch_idx, batch_sentences in split_to_batches(
graphs, self.options.batch_size):
if sentence_idx % print_per == 0 and sentence_idx != 0:
self.logger.print(sentence_idx)
sessions = [self.training_session(sentence, self.logger)
for sentence in batch_sentences]
all_exprs = [next(i) for i in sessions]
if all_exprs:
dn.forward(all_exprs)
all_labels_exprs = [j for i in sessions for j in next(i)]
if all_labels_exprs:
dn.forward(all_labels_exprs)
loss = sum(next(i) for i in sessions) / len(sessions)
self.logger.total_loss_value += loss.value()
if update:
loss.backward()
self.trainer.update()
dn.renew_cg()
sessions.clear()
yield (loss if not update else None)
def train(self, graphs, extra=None):
for _ in self.train_gen(graphs, extra=extra):
pass
def save(self, prefix):
nn.model_save_helper("pickle", prefix, self.network, self.options)
@classmethod
def load(cls, prefix, new_options=None):
"""
:param prefix: model file name prefix
:type prefix: str
:rtype: MaxSubGraphParser
"""
model = dn.Model()
options, savable = nn.model_load_helper(None, prefix, model)
options.__dict__.update(new_options.__dict__)
ret = cls(options, None, (model, savable))
return ret
class MaxSubTreeWithEmptyParser(TreeWithEmptyTrainer):
@classmethod
def add_parser_arguments(cls, arg_parser):
super(MaxSubTreeWithEmptyParser, cls).add_parser_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--decoder",
type=str,
dest="decoder",
default="eisner2nd",
choices=["eisner2nd"])
group.add_argument("--disable-cost-augment",
action="store_false",
dest="cost_augment",
default=True)
group.add_argument("--enable-2nd",
action="store_true",
dest="use_2nd",
default=False)
EdgeEvaluationNetwork.add_parser_arguments(arg_parser)
def __init__(self,
options,
train_sentences=None,
restore_file=None,
statistics=None):
self.model = dn.Model()
random.seed(1)
self.trainer = dn.AdamTrainer(self.model)
self.activation = activations[options.activation]
# self.decoder = decoders[options.decoder]
self.labelsFlag = options.labelsFlag
self.costaugFlag = options.cost_augment
self.options = options
if "func" in options:
del options.func
if restore_file:
self.container, = dn.load(restore_file, self.model)
networks = list(self.container.components)
self.network = networks.pop(0)
self.statistics = statistics
self.has_emptys = len(statistics.emptys) > 0
if self.has_emptys:
self.network_for_emptys = networks.pop(0)
if self.options.use_2nd:
self.network3 = networks.pop(0)
if self.has_emptys:
self.network3_for_emptys_mid = networks.pop(0)
self.network3_for_emptys_out = networks.pop(0)
assert not networks
else:
self.container = nn.Container(self.model)
self.statistics = statistics = StatisticsWithEmpty.from_sentences(
train_sentences)
self.has_emptys = len(statistics.emptys) > 0
self.network = EdgeEvaluationNetwork(self.container, statistics,
options)
if self.has_emptys:
self.network_for_emptys = EdgeEvaluation(
self.container, options)
if options.use_2nd:
self.network3 = EdgeSiblingEvaluation(self.container, options)
if self.has_emptys:
self.network3_for_emptys_mid = EdgeSiblingEvaluation(
self.container, options)
self.network3_for_emptys_out = EdgeSiblingEvaluation(
self.container, options)
def predict(self, sentence):
""" :type sentence: SentenceWithEmpty"""
for iSentence, sentence in enumerate(sentence):
if len(sentence) >= MAX_SENT_SIZE_EMPTY - 1:
logger.info("sent too long...")
heads = [0 for _ in range(len(sentence))]
labels = [None for _ in range(len(sentence))]
yield sentence.to_simple_sentence(heads, labels, set())
continue
lstm_output = self.network.get_lstm_output(sentence)
scores, exprs = self.network.get_complete_scores(lstm_output)
labels = [None for _ in range(len(sentence))]
if self.has_emptys:
scores_ec, exprs_ec = self.network_for_emptys.get_complete_scores(
lstm_output)
else:
scores_ec = -np.ones(
(len(lstm_output), len(lstm_output)), dtype=np.float64)
scores_all = np.stack([scores, scores_ec])
if self.options.use_2nd:
exprs2nd2, scores2nd2, exprs2nd3, scores2nd3 = self.network3.get_complete_scores(
lstm_output)
if self.has_emptys:
_, _, exprs_mid, scores_mid = self.network3_for_emptys_mid.get_complete_scores(
lstm_output, False)
exprs2nd2_ec, scores2nd2_ec, exprs_out, scores_out = self.network3_for_emptys_out.get_complete_scores(
lstm_output)
scores_all_ec = np.stack([scores2nd2, scores2nd2_ec])
heads, emptys = emptyeisner2nd(scores_all, scores_all_ec,
scores2nd3, scores_mid,
scores_out)
else:
heads = eisner2nd(scores, scores2nd2, scores2nd3)
emptys = set()
else:
heads, emptys = empty_eisner_greedy(scores_all)
if self.labelsFlag:
edges = [(head, "_", modifier)
for modifier, head in enumerate(heads[1:], 1)]
for edge, scores_expr in \
zip(edges, self.network.get_label_scores(lstm_output, edges)):
head, _, modifier = edge
labels_scores = scores_expr.value()
labels[modifier] = \
self.network.irels[max(enumerate(labels_scores), key=itemgetter(1))[0]]
dn.renew_cg()
result = sentence.to_simple_sentence(heads, labels, emptys)
if self.options.output_scores:
# extract full edge scores
if self.has_emptys:
scores_list = [
scores, scores_ec, scores2nd2, scores2nd3,
scores2nd2_ec, scores_mid, scores_out
]
else:
scores_list = [
scores, None, scores2nd2, scores2nd3, None, None, None
]
result.comment = [
base64.b64encode(pickle.dumps(scores_list)).decode()
]
yield result
# noinspection PyUnboundLocalVariable
def train_gen(self, sentences, update=True):
eloss = 0.0
mloss = 0.0
eerrors = 0
etotal = 0
start = time.time()
errs = []
lerrs = []
empty_correct_count = 0
empty_gold_total = 0.00000001
empty_pred_total = 0.00000001
for sent_idx, sentence in enumerate(sentences):
if sent_idx % 100 == 0 and sent_idx != 0:
logger.info(
"Processing sentence number: %d, Loss: %.2f,"
"Empty-R: %.2f, Empty-P: %.2f, Errors: %.2f, Time: %.2f",
sent_idx, eloss / etotal,
empty_correct_count / empty_gold_total,
empty_correct_count / empty_pred_total,
(float(eerrors)) / etotal,
time.time() - start)
start = time.time()
eerrors = 0
eloss = 0.0
etotal = 0
empty_gold_total = 0.00000001
empty_pred_total = 0.00000001
empty_correct_count = 0
if len(sentence) >= MAX_SENT_SIZE_EMPTY - 1:
logger.info("sent too long...")
continue
empty_gold_total += len(sentence.empty_nodes)
lstm_output = self.network.get_lstm_output(sentence)
scores, exprs = self.network.get_complete_scores(lstm_output)
if self.has_emptys:
scores_ec, exprs_ec = self.network_for_emptys.get_complete_scores(
lstm_output)
else:
scores_ec = -np.ones(
(len(lstm_output), len(lstm_output)), dtype=np.float64)
gold = [entry.parent_id for entry in sentence]
scores_all = np.stack([scores, scores_ec])
if self.options.use_2nd:
exprs2nd2, scores2nd2, exprs2nd3, scores2nd3 = self.network3.get_complete_scores(
lstm_output)
if self.has_emptys:
_, _, exprs_mid, scores_mid = self.network3_for_emptys_mid.get_complete_scores(
lstm_output, False)
exprs2nd2_ec, scores2nd2_ec, exprs_out, scores_out = \
self.network3_for_emptys_out.get_complete_scores(lstm_output)
scores_all_ec = np.stack([scores2nd2, scores2nd2_ec])
heads, emptys = emptyeisner2nd(
scores_all, scores_all_ec, scores2nd3, scores_mid,
scores_out, sentence if self.costaugFlag else None)
else:
heads = eisner2nd(scores, scores2nd2, scores2nd3,
gold if self.costaugFlag else None)
emptys = set()
else:
heads, emptys = empty_eisner_greedy(
scores_all, sentence if self.costaugFlag else None)
empty_pred_total += len(emptys)
if self.labelsFlag:
edges = [(head, "_", modifier)
for modifier, head in enumerate(gold[1:], 1)]
for edge, r_scores_expr in \
zip(edges, self.network.get_label_scores(lstm_output, edges)):
head, _, modifier = edge
r_scores = r_scores_expr.value()
gold_label_index = self.network.rels[
sentence[modifier].relation]
wrong_label_index = max(((l, scr)
for l, scr in enumerate(r_scores)
if l != gold_label_index),
key=itemgetter(1))[0]
if r_scores[gold_label_index] < r_scores[
wrong_label_index] + 1:
lerrs.append(r_scores_expr[wrong_label_index] -
r_scores_expr[gold_label_index])
e = sum([1 for h, g in zip(heads[1:], gold[1:]) if h != g])
eerrors += e
if e > 0:
loss = [(exprs[h][i] - exprs[g][i])
for i, (h, g) in enumerate(zip(heads, gold))
if h != g] # * (1.0/float(e))
eloss += e
mloss += e
errs.extend(loss)
if self.has_emptys:
empty_incorrect = sentence.empty_nodes.symmetric_difference(
emptys)
empty_correct = sentence.empty_nodes.intersection(emptys)
empty_correct_count += len(empty_correct)
for i in empty_incorrect:
if i not in sentence.empty_nodes:
errs.append(exprs_ec[i.head][i.position])
else:
errs.append(-exprs_ec[i.head][i.position])
if self.options.use_2nd:
# noinspection PyUnboundLocalVariable
biarcs2, biarcs3 = arcs_with_empty_to_biarcs(heads, emptys)
biarcs2_gold, biarcs3_gold = arcs_with_empty_to_biarcs(
gold, sentence.empty_nodes)
for s, t in biarcs2.symmetric_difference(biarcs2_gold):
sign = 1 if (s, t) in biarcs2 else -1
if isinstance(t, int):
errs.append(exprs2nd2[s, t] * sign)
elif isinstance(t, EdgeToEmpty):
errs.append(exprs2nd2_ec[s, t.position] * sign)
for s, m, t in biarcs3.symmetric_difference(biarcs3_gold):
sign = 1 if (s, m, t) in biarcs3 else -1
if isinstance(m, int) and isinstance(t, int):
errs.append(exprs2nd3[s, m, t] * sign)
elif isinstance(m, EdgeToEmpty) and isinstance(t, int):
errs.append(exprs_mid[s, m.position, t] * sign)
elif isinstance(m, int) and isinstance(t, EdgeToEmpty):
errs.append(exprs_out[s, m, t.position] * sign)
else:
raise TypeError
etotal += len(sentence)
if errs or lerrs:
loss = dn.esum(errs + lerrs) # * (1.0/(float(len(errs))))
else:
loss = dn.scalarInput(0.0)
loss_value = loss.scalar_value()
errs = []
lerrs = []
if loss_value != 0.0:
if update:
loss.backward()
self.trainer.update()
dn.renew_cg()
yield (loss if not update else None)
def train(self, sentences):
for _ in self.train_gen(sentences):
pass
def save(self, prefix):
with open(prefix + ".options", "wb") as f:
pickle.dump((self.options, self.statistics), f)
# noinspection PyArgumentList
dn.save(prefix, [self.container])
@classmethod
def load(cls, prefix, new_options=None):
"""
:param prefix: model file name prefix
:type prefix: str
:rtype: MaxSubTreeWithEmptyParser
"""
with open(prefix + ".options", "rb") as f:
options, statistics = pickle.load(f)
options.__dict__.update(new_options.__dict__)
ret = cls(options, None, prefix, statistics)
return ret
class MaxSubTreeParser(TreeParserBase):
@classmethod
def add_parser_arguments(cls, arg_parser):
super(MaxSubTreeParser, cls).add_parser_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--decoder",
type=str,
dest="decoder",
default="eisner2nd",
choices=["eisner2nd"])
group.add_argument("--cost-augment",
action="store_true",
dest="cost_augment",
default=True)
EdgeEvaluationNetwork.add_parser_arguments(arg_parser)
def __init__(self, options, train_sentences=None, restore_file=None):
self.model = dn.Model()
random.seed(1)
self.trainer = dn.AdamTrainer(self.model)
self.activation = activations[options.activation]
self.decoder = decoders[options.decoder]
self.labelsFlag = options.labelsFlag
self.costaugFlag = options.cost_augment
self.options = options
if "func" in options:
del options.func
if restore_file:
self.container, = dn.load(restore_file, self.model)
self.network, self.network3 = self.container.components
else:
self.container = nn.Container(self.model)
statistics = Statistics.from_sentences(train_sentences)
self.network = EdgeEvaluationNetwork(self.container, statistics,
options)
self.network3 = EdgeSiblingEvaluation(self.container, options)
def predict(self, sentence):
def convert_node(node, heads, labels):
return CoNLLUNode(node.id, node.form, node.lemma, node.cpos,
node.pos, node.feats, heads[node.id],
labels[node.id], "_", "_")
for iSentence, sentence in enumerate(sentence):
if len(sentence) >= MAX_SENT_SIZE_NOEMPTY - 1:
logger.info("sent too long...")
heads = [0 for _ in range(len(sentence))]
heads[0] = -1
labels = [None for _ in range(len(sentence))]
yield CoNLLUSentence(
convert_node(node, heads, labels) for node in sentence
if node.id > 0)
continue
lstm_output = self.network.get_lstm_output(sentence)
scores, exprs = self.network.get_complete_scores(lstm_output)
exprs2nd2, scores2nd2, exprs2nd3, scores2nd3 = self.network3.get_complete_scores(
lstm_output)
heads = self.decoder(scores, scores2nd2, scores2nd3)
labels = [None for _ in range(len(sentence))]
if self.labelsFlag:
edges = [(head, "_", modifier)
for modifier, head in enumerate(heads[1:], 1)]
for edge, scores_expr in \
zip(edges, self.network.get_label_scores(lstm_output, edges)):
head, _, modifier = edge
scores = scores_expr.value()
labels[modifier] = \
self.network.irels[max(enumerate(scores), key=itemgetter(1))[0]]
dn.renew_cg()
yield CoNLLUSentence(
convert_node(node, heads, labels) for node in sentence
if node.id > 0)
def train_gen(self, sentences, update=True):
eloss = 0.0
mloss = 0.0
eerrors = 0
etotal = 0
start = time.time()
errs = []
lerrs = []
for sent_idx, sentence in enumerate(sentences):
if len(sentence) >= MAX_SENT_SIZE_NOEMPTY - 1:
logger.info("sent too long...")
continue
if sent_idx % 100 == 0 and sent_idx != 0:
logger.info(
'Processing sentence number: %d, Loss: %.2f, Errors: %.2f, Time: %.2f',
sent_idx, eloss / etotal, (float(eerrors)) / etotal,
time.time() - start)
start = time.time()
eerrors = 0
eloss = 0.0
etotal = 0
lstm_output = self.network.get_lstm_output(sentence)
scores, exprs = self.network.get_complete_scores(lstm_output)
exprs2nd2, scores2nd2, exprs2nd3, scores2nd3 = self.network3.get_complete_scores(
lstm_output)
gold = [entry.parent_id for entry in sentence]
heads = self.decoder(scores, scores2nd2, scores2nd3,
gold if self.costaugFlag else None)
if self.labelsFlag:
edges = [(head, "_", modifier)
for modifier, head in enumerate(gold[1:], 1)]
for edge, r_scores_expr in \
zip(edges, self.network.get_label_scores(lstm_output, edges)):
head, _, modifier = edge
r_scores = r_scores_expr.value()
gold_label_index = self.network.rels[
sentence[modifier].relation]
wrong_label_index = max(((l, scr)
for l, scr in enumerate(r_scores)
if l != gold_label_index),
key=itemgetter(1))[0]
if r_scores[gold_label_index] < r_scores[
wrong_label_index] + 1:
lerrs.append(r_scores_expr[wrong_label_index] -
r_scores_expr[gold_label_index])
e = sum([1 for h, g in zip(heads[1:], gold[1:]) if h != g])
eerrors += e
if e > 0:
loss = [(exprs[h][i] - exprs[g][i])
for i, (h, g) in enumerate(zip(heads, gold))
if h != g] # * (1.0/float(e))
eloss += e
mloss += e
errs.extend(loss)
heads2nd2, heads2nd3 = arcs_to_biarcs(heads)
gold2nd2, gold2nd3 = arcs_to_biarcs(gold)
for i, k in heads2nd2:
errs.append(exprs2nd2[i, k])
for i, k in gold2nd2:
errs.append(-exprs2nd2[i, k])
for i, j, k in heads2nd3:
errs.append(exprs2nd3[i, j, k])
for i, j, k in gold2nd3:
errs.append(-exprs2nd3[i, j, k])
etotal += len(sentence)
if errs or lerrs:
loss = dn.esum(errs + lerrs) # * (1.0/(float(len(errs))))
else:
loss = dn.scalarInput(0.0)
loss_value = loss.scalar_value()
errs = []
lerrs = []
if loss_value != 0.0:
if update:
loss.backward()
self.trainer.update()
dn.renew_cg()
yield (loss if not update else None)
def train(self, sentences):
for _ in self.train_gen(sentences):
pass
def save(self, prefix):
with open(prefix + ".options", "wb") as f:
pickle.dump(self.options, f)
# noinspection PyArgumentList
dn.save(prefix, [self.network])
@classmethod
def load(cls, prefix, new_optons=None):
"""
:param prefix: model file name prefix
:type prefix: str
:rtype: MaxSubGraphParser
"""
with open(prefix + ".options") as f:
options = pickle.load(f)
if new_optons is not None:
options.__dict__.update(new_optons)
ret = cls(options, None, prefix)
return ret
class MaxSubTreeParser(TreeParserBase):
# smaller batch size is better for this task
default_batch_size = 2
default_test_batch_size = 64
@classmethod
def add_parser_arguments(cls, arg_parser):
super(MaxSubTreeParser, cls).add_parser_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--optimizer", type=str, dest="optimizer", default="adam", choices=nn.trainers.keys())
group.add_argument("--decoder", type=str, dest="decoder", default="eisner", choices=decoders)
group.add_argument("--cost-augment", action="store_true", dest="cost_augment", default=True)
group.add_argument("--batch-size", type=int, dest="batch_size", default=cls.default_batch_size)
group.add_argument("--model-format", dest="model_format", choices=nn.model_formats, default="pickle")
EdgeEvaluationNetwork.add_parser_arguments(arg_parser)
@classmethod
def add_predict_arguments(cls, arg_parser):
super(MaxSubTreeParser, cls).add_predict_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__)
group.add_argument("--model-format", dest="model_format", choices=nn.model_formats, default=None)
@classmethod
def add_common_arguments(cls, arg_parser):
super(MaxSubTreeParser, cls).add_common_arguments(arg_parser)
group = arg_parser.add_argument_group(cls.__name__ + " (common)")
group.add_argument("--test-batch-size", type=int, dest="test_batch_size", default=cls.default_test_batch_size)
group.add_argument("--concurrent-count", type=int, dest="concurrent_count", default=5)
def __init__(self, options, train_sentences=None, restore_file=None):
self.model = dn.Model()
if restore_file:
old_options, self.network = nn.model_load_helper(None, restore_file, self.model)
if options is not None:
old_options.__dict__.update(options.__dict__)
options = old_options
else:
statistics = Statistics.from_sentences(train_sentences)
self.network = EdgeEvaluationNetwork(self.model, statistics, options)
self.optimizer = nn.get_optimizer(self.model, options)
self.decoder = decoders[options.decoder]
self.labelsFlag = options.labelsFlag
self.options = options
if "func" in options:
del options.func
def predict_session(self, sentence, pool):
lstm_output = self.network.get_lstm_output(sentence)
length = len(sentence)
raw_exprs = self.network.edge_eval.get_complete_raw_exprs(lstm_output)
yield raw_exprs
scores = self.network.edge_eval.raw_exprs_to_scores(raw_exprs, length)
heads_future = pool.apply_async(self.decoder, (scores, ))
yield None
heads = heads_future.get()
labels = [None for _ in range(len(sentence))]
if self.labelsFlag:
edges = [(head, "_", modifier) for modifier, head in enumerate(heads[1:], 1)]
labels_exprs = list(self.network.label_eval.get_label_scores(lstm_output, edges))
yield labels_exprs
for edge, scores_expr in zip(edges, labels_exprs):
head, _, modifier = edge
edges_scores = scores_expr.value()
labels[modifier] = \
self.network.irels[max(enumerate(edges_scores), key=itemgetter(1))[0]]
else:
yield []
def convert_node(node):
# noinspection PyArgumentList
return CoNLLUNode(node.id, node.form, node.lemma, node.cpos,
node.pos, node.feats,
heads[node.id], labels[node.id],
"_", "_")
result = CoNLLUSentence(convert_node(node) for node in sentence if node.id > 0)
if self.options.output_scores:
# extract full edge scores
edges = [(head, "_", modifier) for head in range(len(sentence))
for modifier in range(len(sentence))]
edges_scores_all = np.array(list(i.value() for i in self.network.get_label_scores(lstm_output, edges)))
edges_scores_all = edges_scores_all.reshape((len(sentence), len(sentence), len(self.network.rels)))
result.comment = [base64.b64encode(pickle.dumps(scores)).decode()]
result.comment.append(base64.b64encode(pickle.dumps(edges_scores_all)).decode()
if self.labelsFlag else "No labels")
yield result
def predict(self, sentences):
self.network.sent_embedding.rnn.disable_dropout()
pool = Pool(self.options.concurrent_count)
for sentence_idx, batch_idx, batch_sentences in split_to_batches(
sentences, self.options.test_batch_size):
sessions = [self.predict_session(sentence, pool)
for sentence in batch_sentences]
all_exprs = [next(i) for i in sessions]
if all_exprs:
dn.forward(all_exprs)
# spawn decoders
for i in sessions:
next(i)
all_labels_exprs = [j for i in sessions for j in next(i)]
if all_labels_exprs:
dn.forward(all_labels_exprs)
for i in sessions:
yield next(i)
dn.renew_cg()
def training_session(self, sentence, print_logger, pool):
lstm_output = self.network.get_lstm_output(sentence)
length = len(sentence)
raw_exprs = self.network.edge_eval.get_complete_raw_exprs(lstm_output)
yield raw_exprs
scores = self.network.edge_eval.raw_exprs_to_scores(raw_exprs, length)
exprs = self.network.edge_eval.raw_exprs_to_exprs(raw_exprs, length)
gold = [entry.parent_id for entry in sentence]
heads_future = pool.apply_async(self.decoder,
(scores, gold if self.options.cost_augment else None))
yield None
heads = heads_future.get()
label_loss = dn.scalarInput(0.0)
if self.labelsFlag:
edges = [(head, "_", modifier) for modifier, head in enumerate(gold[1:], 1)]
label_exprs = list(self.network.get_label_scores(lstm_output, edges))
yield label_exprs
for edge, r_scores_expr in zip(edges, label_exprs):
head, _, modifier = edge
r_scores = r_scores_expr.value()
gold_label_index = self.network.rels[sentence[modifier].relation]
wrong_label_index = max(((l, scr) for l, scr in enumerate(r_scores)
if l != gold_label_index), key=itemgetter(1))[0]
if r_scores[gold_label_index] < r_scores[wrong_label_index] + 1:
label_loss += r_scores_expr[wrong_label_index] - r_scores_expr[gold_label_index] + 1
else:
yield []
head_exprs = [(exprs[h][i] - exprs[g][i] + 1)
for i, (h, g) in enumerate(zip(heads, gold)) if
h != g]
print_logger.correct_edge += len(sentence) - len(head_exprs)
print_logger.total_edge += len(sentence)
head_loss = dn.esum(head_exprs) if head_exprs else dn.scalarInput(0.0)
yield label_loss + head_loss
def train_gen(self, sentences, update=True):
print_logger = PrintLogger()
pool = Pool(self.options.concurrent_count)
self.network.sent_embedding.rnn.set_dropout(self.options.lstm_dropout)
print_per = (100 // self.options.batch_size + 1) * self.options.batch_size
for sentence_idx, batch_idx, batch_sentences in split_to_batches(
sentences, self.options.batch_size):
if sentence_idx % print_per == 0 and sentence_idx != 0:
print_logger.print(sentence_idx)
sessions = [self.training_session(sentence, print_logger, pool)
for sentence in batch_sentences]
all_exprs = [next(i) for i in sessions]
if all_exprs:
dn.forward(all_exprs)
# spawn decoders
for i in sessions:
next(i)
all_labels_exprs = [j for i in sessions for j in next(i)]
if all_labels_exprs:
dn.forward(all_labels_exprs)
loss = sum(next(i) for i in sessions) / len(sessions)
print_logger.total_loss_value += loss.value()
if update:
loss.backward()
self.optimizer.update()
dn.renew_cg()
yield (loss if not update else None)
def train(self, sentences):
for _ in self.train_gen(sentences):
pass
def save(self, prefix):
with open(prefix + ".options", "wb") as f:
pickle.dump(self.options, f)
# noinspection PyArgumentList
dn.save(prefix, [self.network])
@classmethod
def load(cls,
prefix, # type: str
new_options=None):
"""
:param prefix: model file name prefix
:rtype: MaxSubGraphParser
"""
ret = cls(new_options, None, prefix)
return ret