def get_batch(data, pipeline, index=DATASET_SIZE, batch_class=Batch, skip=2, dst=False): """ Parameters ---------- data data to use pipeline : bool or Pipeline whether to get a batch from a dataset or a pipeline index : DatasetIndex batch_class : type skip : int how many batches to skip dst : bool or list of str preload data when False or load to components given """ if dst is False: dataset = Dataset(index, preloaded=data, batch_class=batch_class) else: dataset = Dataset(index, batch_class=batch_class) template_pipeline = ( Pipeline().init_variable('dummy').update( V('dummy'), B.data) # touch batch data to fire preloading ) if isinstance(pipeline, Pipeline): template_pipeline = pipeline + template_pipeline source = ( dataset >> template_pipeline) if pipeline is not False else dataset #skip K batches for _ in range(skip + 1): batch = source.next_batch(10) if dst is not False: batch = batch.load(src=data, dst=dst) return batch
def test_d(size, n_splits): """Test checks for behaviour of D expression in `set_dataset` action. size size of the dataset. n_splits the number if cv folds. """ dataset = Dataset(size) dataset.cv_split(n_splits=n_splits) pipeline = (Pipeline().init_variable('indices', default=[]).update( V('indices', mode='a'), B('indices')[0])) << dataset.CV(C('fold')).train result = list(range(size)) for fold in range(n_splits): pipeline.set_config({'fold': fold}) start = fold * (size // n_splits) end = (fold + 1) * (size // n_splits) for _ in range(2): pipeline.reset('vars') pipeline.run(1) assert pipeline.v('indices') == result[:start] + result[end:]
def _model_setup(data_format): if data_format == 'channels_last': image_shape = (100, 100, 2) elif data_format == 'channels_first': image_shape = (2, 100, 100) size = 50 batch_shape = (size, ) + image_shape images_array = np.random.random(batch_shape) labels_array = np.random.choice(10, size=size) data = images_array, labels_array dataset = Dataset(index=size, batch_class=ImagesBatch, preloaded=data) model_config = { 'inputs': { 'images': { 'shape': image_shape }, 'labels': { 'classes': 10 } }, 'initial_block/inputs': 'images' } return dataset, model_config
def batch(): """ Prepare batch and load same DATA to comp1 and comp2 components. """ dataset = Dataset(BATCH_SIZE, MyBatch) batch = (dataset.next_batch(BATCH_SIZE).load(src=DATA, dst='comp1').load( src=DATA, dst='comp2')) return batch
def test_general_get(named_expr): pipeline = (Dataset(10).pipeline({ 'option': 0 }).init_variable('var').do_nothing(named_expr).run(2, lazy=True)) failed = False try: _ = pipeline.next_batch() except KeyError: failed = True if failed: pytest.fail("Name does not exist")
def test_inbatch_parallel_p(named_expr): """ Check if P() is evalauted properly """ pipeline = (Dataset(10, MyBatch).pipeline( dict(mean=0., std=1., option=ARRAY_INIT)).add_namespace(np).init_variable( 'var', ARRAY_INIT).update(B.images, ARRAY_INIT).update( B.masks, ARRAY_INIT).ip_test(param=P(named_expr)).run(BATCH_SIZE, lazy=True)) _ = pipeline.next_batch() assert True
def test_apply_parallel_p(p_type, named_expr, src): """ Check if P() is evalauted properly """ pipeline = (Dataset(10, MyBatch).pipeline( dict(mean=0., std=1., option=ARRAY_INIT)).add_namespace(np).init_variable( 'var', ARRAY_INIT).update(B.images, ARRAY_INIT).update( B.masks, ARRAY_INIT).ap_test( src=src, param=p_type(named_expr)).run(BATCH_SIZE, lazy=True)) b = pipeline.next_batch() if isinstance(src, str): assert True else: assert (b.images == b.masks).all()
def test_rebatch(batch_size, rebatch_size): """ checks that rebatch produces batches of expected lengths (and doesn't crash)""" data = np.vstack([np.array([i, i]) for i in range(DATASET_SIZE)]) data = (data, ) dataset = Dataset(index=DATASET_SIZE, batch_class=MyBatch, preloaded=data) # workaround for pipeline variables getting lost after rebatch batch_lengths = {'before': [], 'after': []} def get_batch_len(batch, dump): batch_lengths[dump].append(batch.size) p = (Pipeline().call(get_batch_len, 'before').rebatch(rebatch_size).call( get_batch_len, 'after')) << dataset p.run(batch_size=batch_size, n_epochs=1, bar=True) check_batch_lengths(batch_lengths['before'], batch_size) check_batch_lengths(batch_lengths['after'], rebatch_size)
def _model_setup(data_format, image_shape=100): dataset_size = 50 num_classes = 10 if data_format == 'channels_last': image_shape = (image_shape, image_shape, 2) else: image_shape = (2, image_shape, image_shape) batch_shape = (dataset_size, *image_shape) images_array = np.random.random(batch_shape) labels_array = np.random.choice(num_classes, size=dataset_size) data = images_array, labels_array dataset = Dataset(index=dataset_size, batch_class=ImagesBatch, preloaded=data) model_config = { 'inputs/images/shape': image_shape, 'inputs/labels/classes': num_classes, 'initial_block/inputs': 'images' } return dataset, model_config
def test_i(name, expectation, limit_name, limit_value, result): """ Check for behaviour of I under different pipeline configurations. name Name of I, defines its output. expectation Test is expected to raise an error when names requires calculaion of total iterations (e.g. for 'm') and this number is not defined in pipeline (limit_value is None). limit_name 'n_epochs' or 'n_iters' limit_value Total numer of epochs or iteration to run. result Expected output of I. If None, I is expected to raise an error. """ kwargs = {'batch_size': 2, limit_name: limit_value, 'lazy': True} pipeline = (Dataset(10).pipeline().init_variable('var', -1).update( V('var', mode='w'), I(name)).run(**kwargs)) with expectation: _ = pipeline.next_batch() assert pipeline.get_variable('var') == result
def dataset(): index = DatasetIndex(100) return Dataset(index, Batch)