def test_from_json(tmpdir): json_path = json_data(tmpdir) dm = TranslationData.from_json("input", "target", backbone=TEST_BACKBONE, train_file=json_path, batch_size=1) batch = next(iter(dm.train_dataloader())) assert "labels" in batch assert "input_ids" in batch
def test_from_json(tmpdir): json_path = json_data(tmpdir) dm = TranslationData.from_json( "input", "target", train_file=json_path, batch_size=1, ) batch = next(iter(dm.train_dataloader())) assert isinstance(batch[DataKeys.INPUT][0], str) assert isinstance(batch[DataKeys.TARGET][0], str)
def test_from_json_with_field(tmpdir): json_path = json_data_with_field(tmpdir) dm = TranslationData.from_json( "input", "target", backbone=TEST_BACKBONE, train_file=json_path, batch_size=1, field="data", src_lang="en_XX", tgt_lang="ro_RO", ) batch = next(iter(dm.train_dataloader())) assert "labels" in batch assert "input_ids" in batch