def main() -> None: # pylint: disable=no-member,broad-except parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("config", metavar="INI-FILE", help="the configuration file of the experiment") parser.add_argument("datasets", metavar="INI-TEST-DATASETS", help="the configuration of the test datasets") parser.add_argument("-g", "--grid", dest="grid", action="store_true", help="look at the SGE variables for slicing the data") args = parser.parse_args() test_datasets = Configuration() test_datasets.add_argument("test_datasets") test_datasets.add_argument("variables", cond=lambda x: isinstance(x, list)) test_datasets.load_file(args.datasets) test_datasets.build_model() datasets_model = test_datasets.model exp = Experiment(config_path=args.config) exp.build_model() exp.load_variables(datasets_model.variables) if args.grid and len(datasets_model.test_datasets) > 1: raise ValueError("Only one test dataset supported when using --grid") for dataset in datasets_model.test_datasets: if args.grid: if ("SGE_TASK_FIRST" not in os.environ or "SGE_TASK_LAST" not in os.environ or "SGE_TASK_STEPSIZE" not in os.environ or "SGE_TASK_ID" not in os.environ): raise EnvironmentError( "Some SGE environment variables are missing") length = int(os.environ["SGE_TASK_STEPSIZE"]) start = int(os.environ["SGE_TASK_ID"]) - 1 end = int(os.environ["SGE_TASK_LAST"]) - 1 if start + length > end: length = end - start + 1 log("Running grid task {} starting at {} with step {}".format( start // length, start, length)) dataset = dataset.subset(start, length) if exp.config.args.evaluation is None: exp.run_model(dataset, write_out=True) else: exp.evaluate(dataset, write_out=True) for session in exp.config.model.tf_manager.sessions: session.close()
def main() -> None: # pylint: disable=no-member,broad-except parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("config", metavar="INI-FILE", help="the configuration file of the experiment") parser.add_argument("datasets", metavar="INI-FILE", help="the configuration file of the experiment") parser.add_argument("--beam", metavar="BEAM_SIZE", type=int, default=10, help="Beam size.") parser.add_argument("--kenlm", type=str, default=None, help="Path to a KenLM model arpa file.") parser.add_argument("--lm-weight", type=float, help="Weight of the language model.") parser.add_argument("--null-trail-weight", type=float, help="Weight of the null-trailing feature.") parser.add_argument("--nt-ratio-weight", type=float, help="Weight of the null-token ratio feature.") parser.add_argument("--out", type=str, help="Path to the output file.") args = parser.parse_args() test_datasets = Configuration() test_datasets.add_argument("test_datasets") test_datasets.add_argument("batch_size", cond=lambda x: x > 0) test_datasets.add_argument("variables", cond=lambda x: isinstance(x, list)) test_datasets.load_file(args.datasets) test_datasets.build_model() datasets_model = test_datasets.model exp = Experiment(config_path=args.config) exp.build_model() exp.load_variables(datasets_model.variables) ctc_decoder = None for runner in exp.model.runners: if (isinstance(runner, PlainRunner) and isinstance(runner.decoder, CTCDecoder)): ctc_decoder = runner.decoder break if ctc_decoder is None: raise ValueError( "Was not able to detect CTC decoder in the configuration.") logits_runner = RepresentationRunner(output_series="logits", encoder=ctc_decoder, attribute="logits") exp.model.runners = [logits_runner] dataset = datasets_model.test_datasets[0] singleton_batches = dataset.batches(BatchingScheme(1)) print("Loading language model") lm = NGramModel(args.kenlm) print("LM loaded") weights = {} if args.lm_weight: weights['lm_score'] = args.lm_weight if args.null_trail_weight: weights['null_trailing'] = args.null_trail_weight if args.nt_ratio_weight: weights['null_token_ratio'] = args.nt_ratio_weight print("Weights:", weights) i = 0 stats = [] with open(args.out, 'w') as out_file: for sent_dataset in singleton_batches: t1 = timeit.default_timer() ctc_model_result = exp.run_model(sent_dataset, write_out=False, batch_size=1) t2 = timeit.default_timer() logits = np.squeeze(ctc_model_result[1]['logits'], axis=1) t3 = timeit.default_timer() best_hyp = decode_beam(logits, args.beam, ctc_decoder.vocabulary, lm=lm, weights=weights) t4 = timeit.default_timer() stats.append([len(best_hyp.tokens), t2 - t1, t4 - t3]) output = " ".join([best_hyp.tokens][0]) out_file.write(output + "\n") if i % 10 == 0: print("[{}] {}".format(i, output)) i += 1 with open(args.out + ".stats", 'w') as stats_file: for line in stats: stats_file.write("{} {:.3f} {:.3f}\n".format(*line)) for session in exp.config.model.tf_manager.sessions: session.close()
def main() -> None: # pylint: disable=no-member,broad-except parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("config", metavar="INI-FILE", help="the configuration file of the experiment") parser.add_argument("datasets", metavar="INI-FILE", help="the configuration file of the experiment") parser.add_argument("--beam", metavar="BEAM_SIZE", type=int, default=10, help="Beam size.") parser.add_argument("--kenlm", type=str, help="Path to a KenLM model arpa file.") parser.add_argument("--prefix", type=str, help="Path used as a prefix of stored checkpoints.") parser.add_argument("--lm-weight", type=float, help="Default weight of the language model.") parser.add_argument("--null-trail-weight", type=float, help="Default weight of the null-trailing feature.") parser.add_argument("--nt-ratio-weight", type=float, help="Default weight of the null-token ratio feature.") args = parser.parse_args() test_datasets = Configuration() test_datasets.add_argument("test_datasets") test_datasets.add_argument("batch_size", cond=lambda x: x > 0) test_datasets.add_argument("variables", cond=lambda x: isinstance(x, list)) test_datasets.load_file(args.datasets) test_datasets.build_model() datasets_model = test_datasets.model exp = Experiment(config_path=args.config) exp.build_model() exp.load_variables(datasets_model.variables) weights = {} if args.lm_weight is not None: weights['lm_score'] = args.lm_weight if args.null_trail_weight is not None: weights['null_trailing'] = args.null_trail_weight if args.nt_ratio_weight is not None: weights['null_token_ratio'] = args.nt_ratio_weight if not weights: raise ValueError("No default weights specified, nothing to train.") ctc_decoder = None for runner in exp.model.runners: if (isinstance(runner, PlainRunner) and isinstance(runner.decoder, CTCDecoder)): ctc_decoder = runner.decoder break if ctc_decoder is None: raise ValueError( "Was not able to detect CTC decoder in the configuration.") print("Loading language model") lm = NGramModel(args.kenlm) print("LM loaded") logits_runner = RepresentationRunner(output_series="logits", encoder=ctc_decoder, attribute="logits") exp.model.runners = [logits_runner] dataset = datasets_model.test_datasets[0] singleton_batches = dataset.batches(BatchingScheme(1)) DATASET_SIZE = dataset.length CHECKPOINTS = 5 CHECKPOINT_ITERS = int(DATASET_SIZE / CHECKPOINTS) print( "{} sentences in the dataset, checkpoint every {} sentences ({} checkpoints in total)." .format(DATASET_SIZE, CHECKPOINT_ITERS, CHECKPOINTS)) for i, sent_dataset in enumerate(singleton_batches): ctc_model_result = exp.run_model(sent_dataset, write_out=False, batch_size=1) logits = np.squeeze(ctc_model_result[1]['logits'], axis=1) target = ctc_model_result[2]['target'][0] train_weights(logits, args.beam, ctc_decoder.vocabulary, target, weights, lm) print( "[{}] Weights:".format(i + 1), ", ".join([ "{}: {:.3f}".format(key, value) for key, value in weights.items() ])) if i != 0 and (i + 1) % CHECKPOINT_ITERS == 0: with open("{}.{}".format(args.prefix, int(i / CHECKPOINT_ITERS)), "w") as f: for key, value in weights.items(): f.write("{}={:.3f}\n".format(key.upper(), value)) print("\nCheckpoint saved.\n") for session in exp.config.model.tf_manager.sessions: session.close()
class NeuralMonkeyModelWrapper(ModelWrapper): def __init__(self, runs_on_features, config_path, vars_path, data_series="", src_caption_series="", caption_series="", alignments_series="", bs_graph_series="bs_target", name=None): """ caption_series -> GreedyRunner output_series alignments_series -> WordAlignmentRunner output_series bs_graph_series -> BeamSearchRunner output_series """ super(NeuralMonkeyModelWrapper, self).__init__(name, runs_on_features) if not os.path.isfile(config_path): raise ValueError("File {} does not exist.".format(config_path)) self._config_path = config_path self._vars_path = vars_path self._data_series = data_series self._src_caption_series = src_caption_series self._caption_series = caption_series self._alignments_series = alignments_series self._bs_graph_series = bs_graph_series self._exp = Experiment(config_path=config_path) self._exp.build_model() self._exp.load_variables([vars_path]) if self._src_caption_series: self.multimodal = True else: self.multimodal = False def run(self, inputs, src_captions=None): """ Args: inputs: A Numpy Array of inputs (feature, or image arrays). src_captions: A list of string source captions. Returns: A list of dictionaries. Each dictionary contains the keys `caption`, `alignments`, `beam_search_output_graph`. """ n_elems = inputs.shape[0] # enc-dec model (runs on images) if not self.runs_on_features: if self._src_caption_series: # TODO: handle multimodal translation case pass else: ds = Dataset("macaque_data", {self._data_series: lambda: inputs}, {}) # dec-only model (runs on feature maps) else: if self._src_caption_series: # TODO: handle multimodal translation case pass else: ds = Dataset("macaque_data", {self._data_series: lambda: inputs}, {}) _, output_series = self._exp.run_model(dataset=ds, write_out=False) if self._caption_series and self._caption_series in output_series: captions = output_series[self._caption_series] else: captions = [None] * n_elems if self._alignments_series and self._alignments_series in output_series: alignments = output_series[self._alignments_series] # WordAlignmentRunner is incompatible with beam search decoding. if self._bs_graph_series: alignments = None else: alignments = [None] * n_elems if self._bs_graph_series and self._bs_graph_series in output_series: bs_out = output_series[self._bs_graph_series] graphs = [] for b in bs_out: attns = [_transform_alignments(a) for a in b['alignments']] graphs.append( BeamSearchOutputGraph(scores=b['scores'], tokens=b['tokens'], parent_ids=b['parent_ids'], alignments=attns)) hyps = [g.collect_hypotheses() for g in graphs] bs_caps = [h['tokens'] for h in hyps] bs_attns = [h['alignments'] for h in hyps] #bs_attns = [[_transform_alignments(h) for h in hyp['alignments']] for hyp in hyps] else: graphs = [None] * n_elems bs_caps = [None] * n_elems bs_attns = [None] * n_elems results = [] for c, a, bs_g, bs_c, bs_a in zip(captions, alignments, graphs, bs_caps, bs_attns): r = {} for x in [(c, 'caption'), (a, 'alignments')]: if x[0] is not None: if 'greedy' not in r: r['greedy'] = {} r['greedy'][x[1]] = x[0] for x in [(bs_g, 'graph'), (bs_c, 'captions'), (bs_a, 'alignments')]: if x[0] is not None: if 'beam_search' not in r: r['beam_search'] = {} r['beam_search'][x[1]] = x[0] results.append(r) return results
def main() -> None: # pylint: disable=no-member parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("config", metavar="INI-FILE", help="the configuration file of the experiment") parser.add_argument("datasets", metavar="INI-TEST-DATASETS", help="the configuration of the test datasets") parser.add_argument("--json", type=str, help="write the evaluation " "results to this file in JSON format") parser.add_argument("-g", "--grid", dest="grid", action="store_true", help="look at the SGE variables for slicing the data") args = parser.parse_args() datasets_model = load_runtime_config(args.datasets) exp = Experiment(config_path=args.config) exp.build_model() exp.load_variables(datasets_model.variables) if args.grid and len(datasets_model.test_datasets) > 1: raise ValueError("Only one test dataset supported when using --grid") results = [] for dataset in datasets_model.test_datasets: if args.grid: if ("SGE_TASK_FIRST" not in os.environ or "SGE_TASK_LAST" not in os.environ or "SGE_TASK_STEPSIZE" not in os.environ or "SGE_TASK_ID" not in os.environ): raise EnvironmentError( "Some SGE environment variables are missing") length = int(os.environ["SGE_TASK_STEPSIZE"]) start = int(os.environ["SGE_TASK_ID"]) - 1 end = int(os.environ["SGE_TASK_LAST"]) - 1 if start + length > end: length = end - start + 1 log("Running grid task {} starting at {} with step {}" .format(start // length, start, length)) dataset = dataset.subset(start, length) if exp.config.args.evaluation is None: exp.run_model(dataset, write_out=True, batch_size=datasets_model.batch_size) else: eval_result = exp.evaluate(dataset, write_out=True, batch_size=datasets_model.batch_size) results.append(eval_result) if args.json: with open(args.json, "w") as f_out: json.dump(results, f_out) f_out.write("\n") for session in exp.config.model.tf_manager.sessions: session.close()
def main() -> None: # pylint: disable=no-member parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("config", metavar="INI-FILE", help="the configuration file of the experiment") parser.add_argument("datasets", metavar="INI-TEST-DATASETS", help="the configuration of the test datasets") parser.add_argument("-s", "--set", type=str, metavar="SETTING", action="append", dest="config_changes", default=[], help="override an option in the configuration; the " "syntax is [section.]option=value") parser.add_argument("-v", "--var", type=str, metavar="VAR", default=[], action="append", dest="config_vars", help="set a variable in the configuration; the syntax " "is var=value (shorthand for -s vars.var=value)") parser.add_argument("--json", type=str, help="write the evaluation " "results to this file in JSON format") parser.add_argument("-g", "--grid", dest="grid", action="store_true", help="look at the SGE variables for slicing the data") args = parser.parse_args() datasets_model = load_runtime_config(args.datasets) args.config_changes.extend("vars.{}".format(s) for s in args.config_vars) exp = Experiment(config_path=args.config, config_changes=args.config_changes) exp.build_model() exp.load_variables(datasets_model.variables) if args.grid and len(datasets_model.test_datasets) > 1: raise ValueError("Only one test dataset supported when using --grid") results = [] for dataset in datasets_model.test_datasets: if args.grid: if ("SGE_TASK_FIRST" not in os.environ or "SGE_TASK_LAST" not in os.environ or "SGE_TASK_STEPSIZE" not in os.environ or "SGE_TASK_ID" not in os.environ): raise EnvironmentError( "Some SGE environment variables are missing") length = int(os.environ["SGE_TASK_STEPSIZE"]) start = int(os.environ["SGE_TASK_ID"]) - 1 end = int(os.environ["SGE_TASK_LAST"]) - 1 if start + length > end: length = end - start + 1 log("Running grid task {} starting at {} with step {}" .format(start // length, start, length)) dataset = dataset.subset(start, length) if exp.config.args.evaluation is None: exp.run_model(dataset, write_out=True) else: eval_result = exp.evaluate(dataset, write_out=True) results.append(eval_result) if args.json: with open(args.json, "w") as f_out: json.dump(results, f_out) f_out.write("\n") for session in exp.config.model.tf_manager.sessions: session.close()