def test_full_pytorch_example(large_mock_mnist_data, tmpdir): # First, generate mock dataset dataset_url = 'file://{}'.format(tmpdir) mnist_data_to_pycarbon_dataset(tmpdir, dataset_url, mnist_data=large_mock_mnist_data, spark_master='local[1]', carbon_files_count=1) # Next, run a round of training using the pytorce adapting data loader from petastorm.pytorch import DataLoader torch.manual_seed(1) device = torch.device('cpu') model = pytorch_example.Net().to(device) optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.5) transform = TransformSpec(pytorch_example._transform_row, removed_fields=['idx']) with DataLoader(make_carbon_reader('{}/train'.format(dataset_url), reader_pool_type='dummy', num_epochs=1, transform_spec=transform), batch_size=32) as train_loader: pytorch_example.train(model, device, train_loader, 10, optimizer, 1) with DataLoader(make_carbon_reader('{}/test'.format(dataset_url), reader_pool_type='dummy', num_epochs=1, transform_spec=transform), batch_size=100) as test_loader: pytorch_example.evaluation(model, device, test_loader)
def generate_mnist_dataset(small_mock_mnist_data, tmpdir_factory): # Using carbon_files_count to speed up the test path = tmpdir_factory.mktemp('data').strpath dataset_url = 'file://{}'.format(path) mnist_data_to_pycarbon_dataset(path, dataset_url, mnist_data=small_mock_mnist_data, spark_master='local[1]', carbon_files_count=1) return path
def test_full_mxnet_example(large_mock_mnist_data, tmpdir): # First, generate mock dataset dataset_url = 'file://{}'.format(tmpdir) mnist_data_to_pycarbon_dataset(tmpdir, dataset_url, mnist_data=large_mock_mnist_data, spark_master='local[1]', carbon_files_count=1) mxnet_example.mnist_iter_test( dataset_path=dataset_url + "/train", num_epoch=1, batch_size=100 )
def test_full_tf_example_unifeid(large_mock_mnist_data, tmpdir): # First, generate mock dataset dataset_url = 'file://{}'.format(tmpdir) mnist_data_to_pycarbon_dataset(tmpdir, dataset_url, mnist_data=large_mock_mnist_data, spark_master='local[1]', carbon_files_count=1) start = time.time() # Tensorflow train and test tf_example_unified.train_and_test(dataset_url=dataset_url, training_iterations=10, batch_size=10, evaluation_interval=10, start=start)