def test_general_help_model():
PLUGINS = _DictPluginManager()
PLUGINS.set('vergeml.cmd', 'test', CommandTest)
PLUGINS.set('vergeml.model', 'test', ModelTest)
env = Environment(model='test', plugins=PLUGINS)
help = HelpCommand('help', plugins=PLUGINS)
assert GENERAL_HELP_MODEL in help.format_general_help(env)
def test_data_live_loader_meta(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
data:
input:
type: test
cache: none
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_meta(data)
def test_data_disk_loader_num_samples(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
data:
input:
type: test
cache: disk-in
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_num_samples(data)
def _env_from_args(args, trained_model, plugins=PLUGINS):
args = deepcopy(args)
# replace hyphen with underscore for python
args = {k.replace('-', '_'): v for k, v in args.items()}
if trained_model:
args['trained_model'] = trained_model
args['is_global_instance'] = True
args['plugins'] = plugins
env = Environment(**args)
return env
def _env_from_args(args, config, AI, plugins=PLUGINS):
args = deepcopy(args)
# replace hyphen with underscore for python
args = {k.replace('-', '_'):v for k,v in args.items()}
if AI:
args['AI'] = AI
args['config'] = config
args['is_global_instance'] = True
args['plugins'] = plugins
env = Environment(**args)
return env
def test_data_mem_out_loader_ops_num_samples(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
data:
input:
type: test
preprocess:
- op: append
cache: mem
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_num_samples(data)
def test_data_mem_loader_read_samples(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
test-split: 2
val-split: 2
data:
input:
type: test
cache: mem-in
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_read_samples(data)
def test_data_disk_loader_with_ops_meta(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
data:
input:
type: test
preprocess:
- op: append
cache: disk-in
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_meta(data)
def test_training(tmpdir):
d1 = tmpdir.mkdir("p1")
buffer = BufferOutput()
display = Display(stdout=buffer, stderr=buffer)
env = Environment(project_dir=str(d1), display=display)
_name = env.start_training()
callback = env.progress_callback(epochs=10, steps=100)
callback(0, 1, acc=0.56, loss=1.234)
callback(0, 2, acc=0.56, loss=1.234, val_acc=0.77)
callback(1, 1, acc=0.56, loss=1.234)
callback(9, 99, acc=0.78, loss=0.837, val_acc=0.67)
env.end_training(final_results=dict(val_acc=0.77))
assert env.get('results.val_acc') == 0.77
with open(os.path.join(env.AI_dir(), "data.yaml")) as f:
data_yaml = yaml.load(f)
with open(os.path.join(env.stats_dir(), "stats.csv")) as f:
stats_csv = f.read()
assert data_yaml['results']['status'] == 'FINISHED'
assert data_yaml['results']['val_acc'] == 0.77
assert data_yaml['results']['acc'] == 0.78
assert stats_csv == """\
def test_data_disk_out_loader_with_ops(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
test-split: 2
val-split: 2
data:
input:
type: test
preprocess:
- op: append
cache: disk
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_read_samples_transformed(data)
def test_data_disk_loader_with_multiplier_ops_between(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
test-split: 2
val-split: 2
data:
input:
type: test
preprocess:
- op: augment
variants: 2
cache: disk-in
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_read_samples_x2_between(data)
def test_instantiate_model():
PLUGINS = _DictPluginManager()
PLUGINS.set('vergeml.model', 'test-model', ModelTest)
env = Environment(model='test-model', plugins=PLUGINS)
assert isinstance(env.model_plugin, ModelTest) == True