def load(cls, prefix, new_options=None): """ :param prefix: model file name prefix :type prefix: str :rtype: MaxSubTreeWithEmptyParser """ if new_options is None: new_options = AttrDict() model = dn.Model() parser, container = nn.model_load_helper(None, prefix, model) parser.container = container parser.options.__dict__.update(new_options.__dict__) self = parser networks = list(self.container.components) self.network = networks.pop(0) if self.has_emptys: self.network_for_emptys = networks.pop(0) self.label_network_for_emptys = networks.pop(0) if self.options.use_2nd: self.network3 = networks.pop(0) if self.has_emptys: self.network3_for_emptys_mid = networks.pop(0) self.network3_for_emptys_out = networks.pop(0) assert not networks return parser
def load(cls, prefix, new_options=None): """ :param prefix: model file name prefix :type prefix: str :rtype: MaxSubGraphParser """ model = dn.Model() options, savable = nn.model_load_helper(None, prefix, model) options.__dict__.update(new_options.__dict__) ret = cls(options, None, (model, savable)) return ret
def load(cls, prefix, new_options=None): model = dn.Model() ret, savable = nn.model_load_helper("pickle", prefix, model) ret.model = model ret.optimizer = nn.trainers[ret.options.optimizer]( *((ret.model, ret.options.learning_rate ) if ret.options.learning_rate is not None else (ret.model, ))) ret.container = savable ret.sent_embeddings, ret.tag_classification, ret.coarsetag_classification, ret.viterbi_decoder = ret.container.components ret.tag_dict = ret.statistics.leaftags ret.coarse_tag_dict = ret.statistics.coarsetags return ret
def load(cls, prefix, new_options=None): model = dn.Model() model_format = new_options.model_format if new_options is not None else None ret, savable = nn.model_load_helper(model_format, prefix, model) ret.model = model ret.optimizer = nn.trainers[ret.options.optimizer]( *((ret.model, ret.options.learning_rate ) if ret.options.learning_rate is not None else (ret.model, ))) ret.container = savable ret.span_ebd_network, ret.span_eval_network, ret.label_eval_network, \ ret.scorer_network = ret.container.components ret.lemmatizer = WordNetLemmatizer() return ret
def load(cls, prefix, new_options=None): model = dn.Model() ret, savable = nn.model_load_helper("pickle", prefix, model) ret.model = model ret.container = savable ret.sent_embeddings, ret.tag_classification, ret.viterbi_decoder = ret.container.components ret.tag_dict = ret.statistics.supertags ret.optimizer = nn.trainers[ret.options.optimizer]( *((ret.model, ret.options.learning_rate) if ret.options.learning_rate is not None else (ret.model,))) logger.info("Learning rate: {}".format(ret.optimizer.learning_rate)) ret.options.__dict__.update(new_options.__dict__) return ret
def __init__(self, options, data_train): self.model = dn.Model() (span_model_options, statistics), self.container = nn.model_load_helper( options.span_model_format, options.span_model_prefix, self.model) logger.info(statistics) self.statistics = statistics self.options = options self.options.__dict__ = AttrDict( chain(span_model_options.__dict__.items(), options.__dict__.items())) logger.info(pformat(self.options.__dict__)) self.optimizer = nn.trainers[options.optimizer]( *((self.model, options.learning_rate ) if options.learning_rate is not None else (self.model, ))) with open(options.derivations, "rb") as f: self.derivations = pickle.load(f, encoding="latin1") self.hrg_statistics = HRGStatistics.from_derivations(self.derivations) self.span_ebd_network, self.span_eval_network, self.label_eval_network = self.container.components self.scorer_network = self.scorers[options.scorer](self.container, self.hrg_statistics, self.options) with open(options.grammar, "rb") as f: self.grammar = pickle.load( f, encoding="latin1") # type: Mapping[str, Mapping[CFGRule, int]] self.terminal_mapping = defaultdict( Counter) # type: Mapping[str, typing.Counter] for (cfg_lhs, cfg_rhs_list), value in self.grammar.items(): if all(isinstance(i, HLexicon) for i in cfg_rhs_list): self.terminal_mapping[cfg_lhs] += value self.lexicon_mapping = defaultdict( Counter) # type: Mapping[Tuple[HLexicon, int], typing.Counter] for (cfg_lhs, cfg_rhs_list), value in self.grammar.items(): rule_name, main_node_count = cfg_lhs.rsplit("#", 1) main_node_count = int(main_node_count) if all(isinstance(i, HLexicon) for i in cfg_rhs_list): lexicon = cfg_rhs_list[0] self.lexicon_mapping[lexicon, main_node_count] += value self.lemmatizer = WordNetLemmatizer() if options.unlexicalized_rules is not None: with open(options.unlexicalized_rules, "rb") as f: self.unlexicalized_rules = pickle.load(f)
def load(cls, prefix, # type: str new_options=None): """ :param prefix: model file name prefix :rtype: MaxSubGraphParser """ model = dn.Model() parser, savable = nn.model_load_helper(None, prefix, model) parser.options.__dict__.update(new_options.__dict__) parser.model = model parser.container = savable parser.network = parser.container.components[0] parser.optimizer = nn.get_optimizer(model, parser.options) return parser
def main(): parser = ArgumentParser() parser.add_argument("input") parser.add_argument("output") parser.add_argument("--input-type", dest="input_type", choices=nn.model_formats, default=None) parser.add_argument("--output-type", dest="output_type", choices=nn.model_formats, default="pickle") args = parser.parse_args() m = dn.Model() options, savable = nn.model_load_helper(args.input_type, args.input, m) nn.model_save_helper(args.output_type, args.output, savable, options)
def load(cls, prefix, new_options=None): model = dn.Model() model_format = new_options.model_format if new_options is not None else None ret, savable = nn.model_load_helper(model_format, prefix, model) ret.model = model ret.optimizer = nn.trainers[ret.options.optimizer]( *((ret.model, ret.options.learning_rate) if ret.options.learning_rate is not None else (ret.model,))) ret.container = savable ret.span_ebd_network, ret.span_eval_network, ret.label_eval_network, \ ret.scorer_network, ret.edge_eval, ret.category_embedding, ret.category_scorer = ret.container.components ret.lemmatizer = WordNetLemmatizer() if getattr(new_options, "unlexicalized_rules", None) is not None: with open(new_options.unlexicalized_rules, "rb") as f: ret.unlexicalized_rules = pickle.load(f) return ret
def __init__(self, options, train_sentences=None, restore_file=None): self.model = dn.Model() if restore_file: old_options, self.network = nn.model_load_helper(None, restore_file, self.model) if options is not None: old_options.__dict__.update(options.__dict__) options = old_options else: statistics = Statistics.from_sentences(train_sentences) self.network = EdgeEvaluationNetwork(self.model, statistics, options) self.optimizer = nn.get_optimizer(self.model, options) self.decoder = decoders[options.decoder] self.labelsFlag = options.labelsFlag self.options = options if "func" in options: del options.func
def load(cls, prefix, new_options=None): """ :param prefix: model file name prefix :type prefix: str :rtype: MaxSubGraphParser """ model = dn.Model() model_format = new_options.model_format if new_options is not None else None (options, statistics), savable = nn.model_load_helper(model_format, prefix, model) logger.info(statistics) if new_options: options.__dict__.update(new_options.__dict__) ret = cls(options, None, (model, savable)) ret.statistics = statistics ret.label_to_int = statistics.labels.word_to_int ret.int_to_label = statistics.labels.int_to_word ret.create_decoders() return ret