def load(cls, prefix, new_options=None):
"""
:param prefix: model file name prefix
:type prefix: str
:rtype: MaxSubTreeWithEmptyParser
"""
if new_options is None:
new_options = AttrDict()
model = dn.Model()
parser, container = nn.model_load_helper(None, prefix, model)
parser.container = container
parser.options.__dict__.update(new_options.__dict__)
self = parser
networks = list(self.container.components)
self.network = networks.pop(0)
if self.has_emptys:
self.network_for_emptys = networks.pop(0)
self.label_network_for_emptys = networks.pop(0)
if self.options.use_2nd:
self.network3 = networks.pop(0)
if self.has_emptys:
self.network3_for_emptys_mid = networks.pop(0)
self.network3_for_emptys_out = networks.pop(0)
assert not networks
return parser
def load(cls, prefix, new_options=None):
"""
:param prefix: model file name prefix
:type prefix: str
:rtype: MaxSubGraphParser
"""
model = dn.Model()
options, savable = nn.model_load_helper(None, prefix, model)
options.__dict__.update(new_options.__dict__)
ret = cls(options, None, (model, savable))
return ret
def load(cls, prefix, new_options=None):
model = dn.Model()
ret, savable = nn.model_load_helper("pickle", prefix, model)
ret.model = model
ret.optimizer = nn.trainers[ret.options.optimizer](
*((ret.model, ret.options.learning_rate
) if ret.options.learning_rate is not None else (ret.model, )))
ret.container = savable
ret.sent_embeddings, ret.tag_classification, ret.coarsetag_classification, ret.viterbi_decoder = ret.container.components
ret.tag_dict = ret.statistics.leaftags
ret.coarse_tag_dict = ret.statistics.coarsetags
return ret
def load(cls, prefix, new_options=None):
model = dn.Model()
model_format = new_options.model_format if new_options is not None else None
ret, savable = nn.model_load_helper(model_format, prefix, model)
ret.model = model
ret.optimizer = nn.trainers[ret.options.optimizer](
*((ret.model, ret.options.learning_rate
) if ret.options.learning_rate is not None else (ret.model, )))
ret.container = savable
ret.span_ebd_network, ret.span_eval_network, ret.label_eval_network, \
ret.scorer_network = ret.container.components
ret.lemmatizer = WordNetLemmatizer()
return ret
def load(cls, prefix, new_options=None):
model = dn.Model()
ret, savable = nn.model_load_helper("pickle", prefix, model)
ret.model = model
ret.container = savable
ret.sent_embeddings, ret.tag_classification, ret.viterbi_decoder = ret.container.components
ret.tag_dict = ret.statistics.supertags
ret.optimizer = nn.trainers[ret.options.optimizer](
*((ret.model, ret.options.learning_rate)
if ret.options.learning_rate is not None else (ret.model,)))
logger.info("Learning rate: {}".format(ret.optimizer.learning_rate))
ret.options.__dict__.update(new_options.__dict__)
return ret
def __init__(self, options, data_train):
self.model = dn.Model()
(span_model_options,
statistics), self.container = nn.model_load_helper(
options.span_model_format, options.span_model_prefix, self.model)
logger.info(statistics)
self.statistics = statistics
self.options = options
self.options.__dict__ = AttrDict(
chain(span_model_options.__dict__.items(),
options.__dict__.items()))
logger.info(pformat(self.options.__dict__))
self.optimizer = nn.trainers[options.optimizer](
*((self.model, options.learning_rate
) if options.learning_rate is not None else (self.model, )))
with open(options.derivations, "rb") as f:
self.derivations = pickle.load(f, encoding="latin1")
self.hrg_statistics = HRGStatistics.from_derivations(self.derivations)
self.span_ebd_network, self.span_eval_network, self.label_eval_network = self.container.components
self.scorer_network = self.scorers[options.scorer](self.container,
self.hrg_statistics,
self.options)
with open(options.grammar, "rb") as f:
self.grammar = pickle.load(
f,
encoding="latin1") # type: Mapping[str, Mapping[CFGRule, int]]
self.terminal_mapping = defaultdict(
Counter) # type: Mapping[str, typing.Counter]
for (cfg_lhs, cfg_rhs_list), value in self.grammar.items():
if all(isinstance(i, HLexicon) for i in cfg_rhs_list):
self.terminal_mapping[cfg_lhs] += value
self.lexicon_mapping = defaultdict(
Counter) # type: Mapping[Tuple[HLexicon, int], typing.Counter]
for (cfg_lhs, cfg_rhs_list), value in self.grammar.items():
rule_name, main_node_count = cfg_lhs.rsplit("#", 1)
main_node_count = int(main_node_count)
if all(isinstance(i, HLexicon) for i in cfg_rhs_list):
lexicon = cfg_rhs_list[0]
self.lexicon_mapping[lexicon, main_node_count] += value
self.lemmatizer = WordNetLemmatizer()
if options.unlexicalized_rules is not None:
with open(options.unlexicalized_rules, "rb") as f:
self.unlexicalized_rules = pickle.load(f)
def load(cls,
prefix, # type: str
new_options=None):
"""
:param prefix: model file name prefix
:rtype: MaxSubGraphParser
"""
model = dn.Model()
parser, savable = nn.model_load_helper(None, prefix, model)
parser.options.__dict__.update(new_options.__dict__)
parser.model = model
parser.container = savable
parser.network = parser.container.components[0]
parser.optimizer = nn.get_optimizer(model, parser.options)
return parser
def main():
parser = ArgumentParser()
parser.add_argument("input")
parser.add_argument("output")
parser.add_argument("--input-type",
dest="input_type",
choices=nn.model_formats,
default=None)
parser.add_argument("--output-type",
dest="output_type",
choices=nn.model_formats,
default="pickle")
args = parser.parse_args()
m = dn.Model()
options, savable = nn.model_load_helper(args.input_type, args.input, m)
nn.model_save_helper(args.output_type, args.output, savable, options)
def load(cls, prefix, new_options=None):
model = dn.Model()
model_format = new_options.model_format if new_options is not None else None
ret, savable = nn.model_load_helper(model_format, prefix, model)
ret.model = model
ret.optimizer = nn.trainers[ret.options.optimizer](
*((ret.model, ret.options.learning_rate)
if ret.options.learning_rate is not None else (ret.model,)))
ret.container = savable
ret.span_ebd_network, ret.span_eval_network, ret.label_eval_network, \
ret.scorer_network, ret.edge_eval, ret.category_embedding, ret.category_scorer = ret.container.components
ret.lemmatizer = WordNetLemmatizer()
if getattr(new_options, "unlexicalized_rules", None) is not None:
with open(new_options.unlexicalized_rules, "rb") as f:
ret.unlexicalized_rules = pickle.load(f)
return ret
def __init__(self, options, train_sentences=None, restore_file=None):
self.model = dn.Model()
if restore_file:
old_options, self.network = nn.model_load_helper(None, restore_file, self.model)
if options is not None:
old_options.__dict__.update(options.__dict__)
options = old_options
else:
statistics = Statistics.from_sentences(train_sentences)
self.network = EdgeEvaluationNetwork(self.model, statistics, options)
self.optimizer = nn.get_optimizer(self.model, options)
self.decoder = decoders[options.decoder]
self.labelsFlag = options.labelsFlag
self.options = options
if "func" in options:
del options.func
def load(cls, prefix, new_options=None):
"""
:param prefix: model file name prefix
:type prefix: str
:rtype: MaxSubGraphParser
"""
model = dn.Model()
model_format = new_options.model_format if new_options is not None else None
(options,
statistics), savable = nn.model_load_helper(model_format, prefix,
model)
logger.info(statistics)
if new_options:
options.__dict__.update(new_options.__dict__)
ret = cls(options, None, (model, savable))
ret.statistics = statistics
ret.label_to_int = statistics.labels.word_to_int
ret.int_to_label = statistics.labels.int_to_word
ret.create_decoders()
return ret