Python SummarizationData.from_json Exemples

Exemple #1

Fichier : test_data.py Projet : PyTorchLightning/lightning-flash

def test_from_json(tmpdir):
    json_path = json_data(tmpdir)
    dm = SummarizationData.from_json("input",
                                     "target",
                                     backbone=TEST_BACKBONE,
                                     train_file=json_path,
                                     batch_size=1)
    batch = next(iter(dm.train_dataloader()))
    assert "labels" in batch
    assert "input_ids" in batch

Exemple #2

Fichier : test_data.py Projet : aniketmaurya/lightning-flash

def test_from_json(tmpdir):
    json_path = json_data(tmpdir)
    dm = SummarizationData.from_json(
        "input",
        "target",
        train_file=json_path,
        batch_size=1,
    )
    batch = next(iter(dm.train_dataloader()))
    assert isinstance(batch[DataKeys.INPUT][0], str)
    assert isinstance(batch[DataKeys.TARGET][0], str)

Exemple #3

Fichier : test_data.py Projet : stjordanis/lightning-flash

def test_from_json_with_field(tmpdir):
    json_path = json_data_with_field(tmpdir)
    dm = SummarizationData.from_json(
        "input",
        "target",
        backbone=TEST_BACKBONE,
        train_file=json_path,
        batch_size=1,
        field="data",
        src_lang="en_XX",
        tgt_lang="ro_RO",
    )
    batch = next(iter(dm.train_dataloader()))
    assert "labels" in batch
    assert "input_ids" in batch