def test_load_invalid_opener(tmp_cwd):
invalid_script = """
def get_X():
raise NotImplementedError
def get_y():
raise NotImplementedError
"""
import_module('opener', invalid_script)
with pytest.raises(exceptions.InvalidInterface):
load_from_module()
def test_opener_check_folders(tmp_cwd):
script = """
from substratools import Opener
class MyOpener(Opener):
def get_X(self, folders):
assert len(folders) == 5
return 'Xclass'
def get_y(self, folders):
return 'yclass'
def fake_X(self):
return 'fakeX'
def fake_y(self):
return 'fakey'
def get_pred(self, path):
return 'pred'
def save_pred(self, y_pred, path):
return 'pred'
"""
import_module('opener', script)
o = load_from_module()
# create some data folders
data_root_path = o._workspace.data_folder
data_paths = [os.path.join(data_root_path, str(i)) for i in range(5)]
[os.makedirs(p) for p in data_paths]
o.data_folders = data_paths
assert o.get_X() == 'Xclass'
def test_load_opener_from_path(tmp_cwd, valid_opener_code):
dirpath = tmp_cwd / 'myopener'
dirpath.mkdir()
path = dirpath / 'my_opener.py'
path.write_text(valid_opener_code)
o = load_from_module(path=path)
assert o.get_X() == 'X'
def test_predictions_check(tmp_cwd, save_predictions_method_body):
script = f"""
import json
from substratools import Opener
class MyOpener(Opener):
def get_X(self, folder):
return 'X'
def get_y(self, folder):
return list(range(0, 3))
def fake_X(self, n_samples):
return 'Xfake'
def fake_y(self, n_samples):
return [0] * 3
def get_predictions(self, path):
with open(path, 'r') as f:
return json.load(f)
def save_predictions(self, pred, path):
{save_predictions_method_body}
"""
import_module('opener', script)
o = load_from_module()
with pytest.raises(exceptions.MissingFileError):
o.save_predictions({'foo': 'bar'})
def execute(interface=None, sysargs=None):
"""Launch metrics command line interface."""
if not interface:
interface = utils.load_interface_from_module(
'metrics',
interface_class=Metrics,
interface_signature=REQUIRED_FUNCTIONS)
cli = _generate_cli()
sysargs = sysargs if sysargs is not None else sys.argv[1:]
args = cli.parse_args(sysargs)
workspace = MetricsWorkspace(
input_data_folder_paths=args.data_sample_paths,
input_predictions_path=args.input_predictions_path,
log_path=args.log_path,
output_perf_path=args.output_perf_path,
)
opener_wrapper = opener.load_from_module(
path=args.opener_path,
workspace=workspace,
)
utils.configure_logging(path=workspace.log_path, debug_mode=args.debug)
metrics_wrapper = MetricsWrapper(
interface,
workspace=workspace,
opener_wrapper=opener_wrapper,
)
fake_data = args.fake_data or FakeDataMode.from_str(args.fake_data_mode)
n_fake_samples = args.n_fake_samples
return metrics_wrapper.score(
fake_data,
n_fake_samples,
)
def _algo_from_args(args):
workspace = AlgoWorkspace(
input_data_folder_paths=args.data_sample_paths,
input_models_folder_path=args.models_path,
log_path=args.log_path,
output_model_path=args.output_model_path,
output_predictions_path=args.output_predictions_path,
)
utils.configure_logging(workspace.log_path, debug_mode=args.debug)
opener_wrapper = opener.load_from_module(
path=args.opener_path,
workspace=workspace,
)
return AlgoWrapper(
interface,
workspace=workspace,
opener_wrapper=opener_wrapper,
)
def test_load_opener_as_class(tmp_cwd):
script = """
from substratools import Opener
class MyOpener(Opener):
def get_X(self, folders):
return 'Xclass'
def get_y(self, folders):
return 'yclass'
def fake_X(self):
return 'fakeX'
def fake_y(self):
return 'fakey'
def get_pred(self, path):
return 'pred'
def save_pred(self, y_pred, path):
return 'pred'
"""
import_module('opener', script)
o = load_from_module()
assert o.get_X() == 'Xclass'
def test_load_opener_as_module(tmp_cwd):
script = """
def _helper():
pass
def get_X(folders):
return 'X'
def get_y(folders):
return 'y'
def fake_X():
return 'fakeX'
def fake_y():
return 'fakey'
def get_pred(path):
return 'pred'
def save_pred(y_pred, path):
return 'pred'
"""
import_module('opener', script)
o = load_from_module()
assert o.get_X() == 'X'
def __init__(self, interface):
self._opener_wrapper = opener.load_from_module()
self._workspace = workspace.Workspace()
self._interface = interface
def __init__(self, interface):
assert isinstance(interface, Algo)
self._opener_wrapper = opener.load_from_module()
self._interface = interface
self._workspace = workspace.Workspace()
def test_load_opener_not_found(tmp_cwd):
with pytest.raises(ImportError):
load_from_module()
def __init__(self, interface, workspace=None, opener_wrapper=None):
assert isinstance(interface, self._INTERFACE_CLASS)
self._workspace = workspace or self._DEFAULT_WORKSPACE_CLASS()
self._opener_wrapper = opener_wrapper or \
opener.load_from_module(workspace=self._workspace)
self._interface = interface
def __init__(self, interface, workspace=None, opener_wrapper=None):
self._workspace = workspace or MetricsWorkspace()
self._opener_wrapper = opener_wrapper or \
opener.load_from_module(workspace=self._workspace)
self._interface = interface
