def prepare(input_root, output_file_path, split, droph, radius):
# Function for input and output path
out_path = lambda f: os.path.join(output_file_path, f)
inp_path = lambda f: os.path.join(input_root, f)
# Define which elements to drop
drop = []
if droph: drop.append('H')
# Logger
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format='%(asctime)s %(levelname)s %(process)d: ' +
'%(message)s')
if split: # use the split datasets
logger.info(f'Processing datasets from {input_root:}.')
# Training set
logger.info(f'Processing training dataset...')
dataset = da.LMDBDataset(inp_path('train'),
transform=EnvironmentSelection(radius))
_save_dict = _write_npz(dataset, out_path('train.npz'), drop)
# Validation set
logger.info(f'Processing validation dataset...')
dataset = da.LMDBDataset(inp_path('val'),
transform=EnvironmentSelection(radius))
_save_dict = _write_npz(dataset, out_path('valid.npz'), drop)
# Test set
logger.info(f'Processing test dataset...')
dataset = da.LMDBDataset(inp_path('test'),
transform=EnvironmentSelection(radius))
_save_dict = _write_npz(dataset, out_path('test.npz'), drop)
else: # use the full data set
logger.info(f'Processing full dataset from {input_root:}...')
dataset = da.LMDBDataset(inp_path('all'),
transform=EnvironmentSelection(radius))
_save_dict = _write_npz(dataset, out_path('all.npz'), drop)
def prepare(input_root, output_file_path, split):
# Logger
logging.basicConfig(stream=sys.stdout,
format='%(asctime)s %(levelname)s %(process)d: ' +
'%(message)s',
level=logging.INFO)
if split:
logger.info(f'Processing datasets from {input_root:}.')
logger.info(f'Processing training dataset...')
dataset = da.LMDBDataset(os.path.join(input_root, 'train'))
_write_npz(dataset, os.path.join(output_file_path, 'train.npz'))
logger.info(f'Processing validation dataset...')
dataset = da.LMDBDataset(os.path.join(input_root, 'val'))
_write_npz(dataset, os.path.join(output_file_path, 'valid.npz'))
logger.info(f'Processing test dataset from...')
dataset = da.LMDBDataset(os.path.join(input_root, 'test'))
_write_npz(dataset, os.path.join(output_file_path, 'test.npz'))
else:
logger.info(f'Processing full dataset from {input_root:}...')
dataset = da.LMDBDataset(os.path.join(input_root, 'all'))
_write_npz(dataset, os.path.join(output_file_path, 'all.npz'))