def pipeline():
""" Creates a pipeline configured to use a given model with a specified configuration.
Notes
-----
Pipeline can be executed only if its config contains the following parameters:
model_class : TFModel
Architecture of model. List of available models is defined at 'AVAILABLE_MODELS'.
model_config : Config
Model parameters.
Returns
-------
Pipeline
A pipeline that contains model initialization and training with a given config.
"""
test_pipeline = (Pipeline().init_variable('current_loss').init_model(
'dynamic', C('model_class'), 'model',
C('model_config')).to_array().train_model('model',
fetches='loss',
images=B('images'),
labels=B('labels'),
save_to=V('current_loss')))
return test_pipeline
def _pipelines(model_class):
config = {}
data_format = predict_args = predict_kwargs = None
if issubclass(model_class, TFModel):
data_format = 'channels_last'
config.update({'dtype': None})
predict_args = ()
predict_kwargs = dict(images=B('images'))
elif issubclass(model_class, TorchModel):
data_format = 'channels_first'
config.update({'dtype': 'float32'})
predict_args = (B('images'),)
predict_kwargs = dict()
dataset, model_config = model_setup_images_clf(data_format)
config.update({'model_class': model_class, 'model_config': model_config})
save_pipeline = (Pipeline()
.init_variable('predictions', default=[])
.init_model('dynamic', C('model_class'), 'model', C('model_config'))
.to_array(dtype=C('dtype'))
.predict_model('model', *predict_args,
fetches='predictions', save_to=V('predictions', mode='a'),
**predict_kwargs))
load_pipeline = (Pipeline()
.init_variable('predictions', default=[])
.to_array(dtype=C('dtype'))
.predict_model('model', *predict_args,
fetches='predictions', save_to=V('predictions', mode='a'),
**predict_kwargs))
save_pipeline = (save_pipeline << dataset) << config
load_pipeline = (load_pipeline << dataset) << config
return save_pipeline, load_pipeline
def model_pipeline():
""" Creates instance of Pipeline that is configured to use given model
with passed parameters.
Parameters
----------
model_class : subclass of TFModel
Architecture of model. List of available models is defined at 'AVAILABLE_MODELS'.
current_config : dict
Dictionary with parameters of model.
Returns
-------
Pipeline
Test pipeline that consists of initialization of model and
preparing for training with given config.
"""
test_pipeline = (Pipeline().init_variable('current_loss').init_model(
'dynamic', C('model_class'), 'TestModel',
C('model_config')).to_array().train_model('TestModel',
fetches='loss',
images=B('images'),
labels=B('labels'),
save_to=V('current_loss')))
return test_pipeline
def test_d(size, n_splits):
"""Test checks for behaviour of D expression in `set_dataset` action.
size
size of the dataset.
n_splits
the number if cv folds.
"""
dataset = Dataset(size)
dataset.cv_split(n_splits=n_splits)
pipeline = (Pipeline().init_variable('indices', default=[]).update(
V('indices', mode='a'),
B('indices')[0])) << dataset.CV(C('fold')).train
result = list(range(size))
for fold in range(n_splits):
pipeline.set_config({'fold': fold})
start = fold * (size // n_splits)
end = (fold + 1) * (size // n_splits)
for _ in range(2):
pipeline.reset('vars')
pipeline.run(1)
assert pipeline.v('indices') == result[:start] + result[end:]
def test_run(self, save_path, pipelines, model_class):
"""
Check model loading and saving during pipeline iterations
A model is initialised in save_pipeline, then for each batch:
predictions are obtained and saved;
current model state is saved;
the model is trained.
After that in load_pipeline for each batch in same dataset:
the model from corresponding iteration is loaded;
predictions are obtained and saved.
Predictions from save_pipeline and from load_pipeline should be equal
"""
save_pipeline, load_pipeline = pipelines(model_class)
train_args, train_kwargs = self.train_args(model_class)
save_tmpl = (Pipeline()
.save_model('model', path=save_path + I("current").str())
.train_model('model', *train_args, **train_kwargs))
save_pipeline = save_pipeline + save_tmpl
save_pipeline.run(BATCH_SIZE, n_epochs=1, bar=True)
saved_predictions = save_pipeline.get_variable('predictions')
load_tmpl = (Pipeline()
.load_model('dynamic', C('model_class'), 'model', path=save_path + I("current").str()))
load_pipeline = load_tmpl + load_pipeline
load_pipeline.run(BATCH_SIZE, n_epochs=1, bar=True)
loaded_predictions = load_pipeline.get_variable('predictions')
assert (np.concatenate(saved_predictions) == np.concatenate(loaded_predictions)).all()
def test_save_to_c(self):
pipeline = Pipeline(config=Config(some=100))
save_data_to(data=200, dst=C('value'), pipeline=pipeline)
assert pipeline.config['some'] == 100
assert pipeline.config['value'] == 200
def test_save_to_list(self):
arr = np.zeros(3)
pipeline = Pipeline(config=Config(some=100))
save_data_to(data=[[1, 2, 3], 200],
dst=[arr, C('value')],
pipeline=pipeline)
assert (arr == [1, 2, 3]).all()
assert pipeline.config['value'] == 200
def test_after_before(self, save_path, pipelines, model_class):
"""
Test model saving in pipeline.after and loading in pipeline.before
"""
save_pipeline, load_pipeline = pipelines(model_class)
save_pipeline.after.save_model('model', path=save_path)
save_pipeline.run(BATCH_SIZE, n_epochs=1)
saved_predictions = save_pipeline.get_variable('predictions')
load_pipeline.before.load_model('dynamic', C('model_class'), 'model', path=save_path)
load_pipeline.run(BATCH_SIZE, n_epochs=1)
loaded_predictions = load_pipeline.get_variable('predictions')
assert (np.concatenate(saved_predictions) == np.concatenate(loaded_predictions)).all()
def test_now(self, save_path, pipelines, model_class):
"""
Test model loading and saving with `save_model_now` and `load_model_now`
"""
save_pipeline, load_pipeline = pipelines(model_class)
save_pipeline.run(BATCH_SIZE, n_epochs=1)
saved_predictions = save_pipeline.get_variable('predictions')
save_pipeline.save_model_now('model', path=save_path)
load_pipeline.load_model_now('dynamic', C('model_class'), 'model', path=save_path)
load_pipeline.run(BATCH_SIZE, n_epochs=1)
loaded_predictions = load_pipeline.get_variable('predictions')
assert (np.concatenate(saved_predictions) == np.concatenate(loaded_predictions)).all()
import sys
from contextlib import ExitStack as does_not_raise
import pytest
import numpy as np
sys.path.append('..')
from batchflow import B, C, D, F, L, V, R, P, I, Dataset, Pipeline, Batch, apply_parallel, inbatch_parallel, action
#--------------------
# COMMON
#--------------------
@pytest.mark.parametrize('named_expr', [
C('option'),
C('not defined', default=10),
B('size'),
D('size'),
V('var'),
R('normal', 0, 1),
R('normal', 0, 1, size=B.size),
F(lambda batch: 0),
L(lambda: 0),
])
def test_general_get(named_expr):
pipeline = (Dataset(10).pipeline({
'option': 0
}).init_variable('var').do_nothing(named_expr).run(2, lazy=True))
failed = False
# pylint: disable=redefined-outer-name, missing-docstring, bad-continuation
import sys
from contextlib import ExitStack as does_not_raise
import pytest
sys.path.append('..')
from batchflow import B, C, D, F, L, V, R, P, I, Dataset
@pytest.mark.parametrize('named_expr', [
C('option'),
B('size'),
D('size'),
V('var'),
R('normal', 0, 1),
P('normal', 0, 1),
F(lambda batch: 0),
L(lambda: 0),
])
def test_general_get(named_expr):
pipeline = (Dataset(10).pipeline({
'option': 0
}).init_variable('var').do_nothing(named_expr).run(2, lazy=True))
failed = False
try:
_ = pipeline.next_batch()
except KeyError:
failed = True
if failed:
import sys
sys.path.append("../../..")
from batchflow import Pipeline, B, C, V, D
from batchflow.opensets import MNIST
from batchflow.models.tf import VGG16
from batchflow.research import Research
BATCH_SIZE = 64
model_config = {
'inputs/images/shape': (28, 28, 1),
'inputs/labels/classes': D('num_classes'),
'initial_block/inputs': 'images',
'body/block/layout': 'cna',
'device': C('device') # it's technical parameter for TFModel
}
mnist = MNIST()
train_root = mnist.train.p.run(BATCH_SIZE,
shuffle=True,
n_epochs=None,
lazy=True)
test_root = mnist.test.p.run(BATCH_SIZE, shuffle=True, n_epochs=1, lazy=True)
train_template = (Pipeline().init_variable(
'loss', init_on_each_run=list).init_variable(
'accuracy', init_on_each_run=list).init_model(
'dynamic', VGG16, 'conv',
config=model_config).to_array().train_model('conv',
images=B('images'),
import sys
sys.path.append("../../..")
from batchflow import Pipeline, B, C, V, D
from batchflow.opensets import MNIST
from batchflow.models.torch import VGG16
from batchflow.research import Research
BATCH_SIZE = 64
model_config = {
'inputs/images/shape': (1, 28, 28),
'inputs/labels/classes': D('num_classes'),
'initial_block/inputs': 'images',
'body/block/layout': 'cna',
'device': C('device') # it's technical parameter for TFModel
}
mnist = MNIST()
train_root = mnist.train.p.run(BATCH_SIZE,
shuffle=True,
n_epochs=None,
lazy=True)
test_root = mnist.test.p.run(BATCH_SIZE, shuffle=True, n_epochs=1, lazy=True)
train_template = (Pipeline().init_variable(
'loss', init_on_each_run=list).init_variable(
'accuracy',
init_on_each_run=list).init_model('dynamic',
VGG16,
'conv',
# pylint: disable=redefined-outer-name, missing-docstring, bad-continuation
import sys
from contextlib import ExitStack as does_not_raise
import pytest
sys.path.append('..')
from batchflow import B, C, D, F, L, V, R, P, I, Dataset, Pipeline
@pytest.mark.parametrize('named_expr', [
C('option'),
C('not defined', default=10),
B('size'),
D('size'),
V('var'),
R('normal', 0, 1),
P(R('normal', 0, 1)),
F(lambda batch: 0),
L(lambda: 0),
])
def test_general_get(named_expr):
pipeline = (Dataset(10).pipeline({
'option': 0
}).init_variable('var').do_nothing(named_expr).run(2, lazy=True))
failed = False
try:
_ = pipeline.next_batch()
except KeyError:
failed = True