def load_batches():
client = PlatformClient(API_SERVER, EMAIL, PASSWORD)
client.create_dataset(
DATASET_BATCHES,
description="CIFAR-10 batches from"
" http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz")
training_files = [
'data_batch_1', 'data_batch_2', 'data_batch_3', 'data_batch_4',
'data_batch_5'
]
test_files = ['test_batch']
for filename in training_files:
filepath = os.path.join('cifar-10-batches-py/', filename)
print('Loading training file', filepath)
start = time.time()
split_class = 'training'
with open(filepath, 'rb') as f:
client.create_sample(
filename,
DATASET_BATCHES,
data_url=f'gs://elvo-platform/test/platform/data'
f'/{DATASET_BATCHES}/{filename}.pkl',
data_content=f,
split=split_class)
end = time.time()
print(f'Took {end - start} seconds', flush=True)
for filename in test_files:
filepath = os.path.join('cifar-10-batches-py/', filename)
print('Loading test file', filepath)
start = time.time()
split_class = 'test'
with open(filepath, 'rb') as f:
client.create_sample(
filename,
DATASET_BATCHES,
data_url=f'gs://elvo-platform/test/platform/data'
f'/{DATASET_BATCHES}/{filename}.pkl',
data_content=f,
split=split_class)
end = time.time()
print(f'Took {end - start} seconds', flush=True)
API_SERVER = ''
EMAIL = ''
PASSWORD = ''
DATASET = ''
if __name__ == '__main__':
client = PlatformClient(API_SERVER, EMAIL, PASSWORD)
print(f'Creating dataset: {DATASET}')
client.create_dataset(DATASET,
description='First version of the multiphase'
' segmentation data')
root_dir = pathlib.Path(
'/research/rih-cs/datasets/elvo-multiphase/segmentation_data')
for dirpath in root_dir.iterdir():
for filepath in dirpath.iterdir():
if filepath.name.endswith('.jpg'):
sample_name = filepath.name[:-len('.jpg')]
label = sample_name[0] # either 'P' or 'N'
url = f'gs://elvo-platform/multiphase/processed' \
f'/{DATASET}/{filepath.name}'
print(f'Uploading sample {sample_name} with label {label} to'
f' {url} from {str(filepath)}')
with open(filepath, 'rb') as f:
client.create_sample(sample_name,
DATASET,
data_url=url,
data_content=f,
label=label)