class TestParallelTextClassifier(TestCase): def setUp(self): src_voc = Vocabulary() trg_voc = Vocabulary() for tok in "I am Philip".split(): src_voc[tok] for tok in "私 は フィリップ です".split(): trg_voc[tok] self.model = ParallelTextClassifier(Args(), src_voc, trg_voc, optimizer=optimizers.SGD()) self.src_voc = src_voc self.trg_voc = trg_voc def test_read_write(self): model = "/tmp/chainer-test/text/model.temp" X, Y = self.src_voc, self.trg_voc # Train with 1 example inp = np.array([[X["Philip"], X["I"]]], dtype=np.int32) out = np.array([[Y["フィリップ"], Y["私"]]], dtype=np.int32) self.model.train(inp, out) # Save serializer = ModelSerializer(model) serializer.save(self.model) # Load model1 = ParallelTextClassifier(InitArgs(model)) # Check self.assertModelEqual(self.model._model, model1._model)
def setUp(self): src_voc = Vocabulary() trg_voc = Vocabulary() for tok in "I am Philip".split(): src_voc[tok] for tok in "私 は フィリップ です".split(): trg_voc[tok] self.model = ParallelTextClassifier(Args(), src_voc, trg_voc, optimizer=optimizers.SGD()) self.src_voc = src_voc self.trg_voc = trg_voc
parser.add_argument("--init_model", type=str, help="Directory to the model trained with train-nmt.", required=True) # Options parser.add_argument("--batch", type=positive, default=512, help="Number of source word in the batch.") parser.add_argument("--src", type=str, help="Specify this to do batched decoding, it has a priority than stdin.") parser.add_argument("--use_cpu", action="store_true") parser.add_argument("--gpu", type=int, default=-1, help="Which GPU to use (Negative for cpu).") parser.add_argument("--verbose", action="store_true") args = parser.parse_args() """ Sanity Check """ if args.use_cpu: args.gpu = -1 # Loading model UF.trace("Setting up classifier") model = ParallelTextClassifier(args, use_gpu=args.gpu, collect_output=True) SRC, TRG = model.get_vocabularies() # Testing callbacks def print_result(ctr, trg, TRG, src, SRC, fp=sys.stderr): for i, (sent, result) in enumerate(zip(src, trg.y)): print(ctr + i, file=fp) print("INP:", SRC.str_rpr(sent), file=fp) print("TAG:", TRG.str_rpr(result), file=fp) def onDecodingStart(): UF.trace("Tagging started.") def onBatchUpdate(ctr, src, trg): # Decoding if args.verbose: