def load_batches(): client = PlatformClient(API_SERVER, EMAIL, PASSWORD) client.create_dataset( DATASET_BATCHES, description="CIFAR-10 batches from" " http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz") training_files = [ 'data_batch_1', 'data_batch_2', 'data_batch_3', 'data_batch_4', 'data_batch_5' ] test_files = ['test_batch'] for filename in training_files: filepath = os.path.join('cifar-10-batches-py/', filename) print('Loading training file', filepath) start = time.time() split_class = 'training' with open(filepath, 'rb') as f: client.create_sample( filename, DATASET_BATCHES, data_url=f'gs://elvo-platform/test/platform/data' f'/{DATASET_BATCHES}/{filename}.pkl', data_content=f, split=split_class) end = time.time() print(f'Took {end - start} seconds', flush=True) for filename in test_files: filepath = os.path.join('cifar-10-batches-py/', filename) print('Loading test file', filepath) start = time.time() split_class = 'test' with open(filepath, 'rb') as f: client.create_sample( filename, DATASET_BATCHES, data_url=f'gs://elvo-platform/test/platform/data' f'/{DATASET_BATCHES}/{filename}.pkl', data_content=f, split=split_class) end = time.time() print(f'Took {end - start} seconds', flush=True)
API_SERVER = '' EMAIL = '' PASSWORD = '' DATASET = '' if __name__ == '__main__': client = PlatformClient(API_SERVER, EMAIL, PASSWORD) print(f'Creating dataset: {DATASET}') client.create_dataset(DATASET, description='First version of the multiphase' ' segmentation data') root_dir = pathlib.Path( '/research/rih-cs/datasets/elvo-multiphase/segmentation_data') for dirpath in root_dir.iterdir(): for filepath in dirpath.iterdir(): if filepath.name.endswith('.jpg'): sample_name = filepath.name[:-len('.jpg')] label = sample_name[0] # either 'P' or 'N' url = f'gs://elvo-platform/multiphase/processed' \ f'/{DATASET}/{filepath.name}' print(f'Uploading sample {sample_name} with label {label} to' f' {url} from {str(filepath)}') with open(filepath, 'rb') as f: client.create_sample(sample_name, DATASET, data_url=url, data_content=f, label=label)