def test_validate_preprocess(): PLUGINS = _DictPluginManager() PLUGINS.set('vergeml.operation', 'augment', AugmentOperation) VALIDATORS = {'data': ValidateData(plugins=PLUGINS)} apply_config({'data': { 'preprocess': [{ 'op': 'augment', 'variants': 4 }] }}, VALIDATORS) assert VALIDATORS['data'].values == { 'data': { 'cache': '*auto*', 'input': { 'type': None }, 'output': { 'type': None }, 'preprocess': [{ 'op': 'augment', 'variants': 4 }] } }
def set_defaults(self, cmd, args, plugins=PLUGINS): if self.model: self.model.set_defaults(cmd, args, self) validators = dict(device=ValidateDevice('device', plugins), data=ValidateData('data', plugins)) config = dict(device=self.get('device'), data=self.get('data')) apply_config(config, validators) # update env from validators for _, plugin in validators.items(): for k, v in plugin.values.items(): self._config[k] = v
def test_validate_preprocess_invalid(): PLUGINS = _DictPluginManager() PLUGINS.set('vergeml.operation', 'augment', AugmentOperation) VALIDATORS = {'data': ValidateData(plugins=PLUGINS)} with pytest.raises(VergeMLError, match=r".*Did you mean 'variants'.*"): apply_config( {'data': { 'preprocess': [{ 'op': 'augment', 'variantz': 4 }] }}, VALIDATORS)
def _load_and_configure(file, label, validators): doc = load_yaml_file(file, label) try: doc = apply_config(doc, validators) if 'random-seed' in doc and not isinstance(doc['random-seed'], int): raise VergeMLError('Invalid value option random-seed.', 'random-seed must be an integer value.', hint_type='value', hint_key='random-seed') except VergeMLError as e: if e.hint_key: key, kind = e.hint_key, e.hint_type with open(file) as f: definition = yaml_find_definition(f, key, kind) if definition: line, column, length = definition message = display_err_in_file(file, line, column, str(e), length) e.message = message # clear suggestion because it is already contained in the formatted error message. e.suggestion = None raise e else: raise e else: raise e return doc
def test_input_output(): PLUGINS = _DictPluginManager() PLUGINS.set('vergeml.io', 'image', ImageSource) VALIDATORS = {'data': ValidateData('image', plugins=PLUGINS)} apply_config( {'data': { 'input': { 'type': 'image' }, 'output': { 'type': 'image' } }}, validators=VALIDATORS) assert VALIDATORS['data'].values['data']['input']['type'] == 'image' assert VALIDATORS['data'].values['data']['output']['type'] == 'image'
def test_apply_empty_config(): VALIDATORS = {'device': ValidateDevice()} assert apply_config({}, VALIDATORS) == {} assert VALIDATORS['device'].values == { 'device': { 'id': 'auto', 'memory': 'auto', 'grow-memory': False } }
def test_apply_config_image_invalid(): PLUGINS = _DictPluginManager() PLUGINS.set('vergeml.io', 'image', ImageSource) VALIDATORS = {'data': ValidateData(plugins=PLUGINS)} with pytest.raises(VergeMLError): assert apply_config( {'data': { 'input': { 'type': 'image', 'input-patternz': '*.jpg' } }}, VALIDATORS) == {}
def test_apply_config(): VALIDATORS = {'device': ValidateDevice()} assert apply_config({ 'device': 'gpu', 'model': 'inception-v3' }, VALIDATORS) == { 'model': 'inception-v3' } assert VALIDATORS['device'].values == { 'device': { 'id': 'gpu:0', 'memory': 'auto', 'grow-memory': False } }
def test_apply_config_image(): PLUGINS = _DictPluginManager() PLUGINS.set('vergeml.io', 'image', ImageSource) VALIDATORS = {'data': ValidateData(plugins=PLUGINS)} assert apply_config( {'data': { 'input': { 'type': 'image', 'input-patterns': '*.jpg' } }}, VALIDATORS) == {} assert VALIDATORS['data'].values == { 'data': { 'input': { 'type': 'image', 'input-patterns': '*.jpg' }, 'output': { 'type': None }, 'cache': '*auto*', 'preprocess': [] } }
def __init__(self, model=None, project_file=None, samples_dir=None, test_split=None, val_split=None, cache_dir=None, random_seed=None, trainings_dir=None, project_dir=None, AI=None, is_global_instance=False, config={}, plugins=PLUGINS, display=DISPLAY): """Configure, train and save the results. :param model: Name of the model plugin. :param project_file: Optional path to the project file. :param samples_dir: The directory where samples can be found. [default: samples] :param test_split: The test split. [default: 10%] :param val_split: The val split. [default: 10%] :param cache_dir: The directory used for caching [default: .cache] :param random_seed: Random seed. [default 2204] :param trainings_dir: The directory to save training results to. [default: trainings] :param project_dir: The directory of the project. [default: current directory] :param AI: Optional name of a trained AI. :is_global_instance: If true, this env can be accessed under the global var env.ENV. [default: false] :config: Additional configuration to pass to env, i.e. if not using a project file """ super().__init__() # when called from the command line, we need to have a global instance if is_global_instance: global ENV ENV = self # setup the display self.display = display # set the name of the AI if given self.AI = AI # this holds the model object (not the name of the model) self.model = None # the results class (responsible for updating data.yaml with the latest results during training) self.results = None # when a training is started, this holds the object responsible for coordinating the training self.training = None # hold a proxy to the data loader self._data = None self.plugins = plugins # set up the base options from constructor arguments self._config = {} self._config['samples-dir'] = samples_dir self._config['test-split'] = test_split self._config['val-split'] = val_split self._config['cache-dir'] = cache_dir self._config['random-seed'] = random_seed self._config['trainings-dir'] = trainings_dir self._config['model'] = model validators = {} # add validators for commands for k, v in plugins.all('vergeml.cmd').items(): cmd = Command.discover(v) validators[cmd.name] = ValidateOptions(cmd.options, k, plugins=plugins) # now it gets a bit tricky - we need to peek at the model name # to find the right validators to create for model commands. peek_model_name = model peek_trainings_dir = trainings_dir # to do this, we have to first have a look at the project file try: project_doc = load_yaml_file(project_file) if project_file else {} # only update model name if empty (project file does not override command line) peek_model_name = peek_model_name or project_doc.get('model', None) # pick up trainings-dir in the same way peek_trainings_dir = peek_trainings_dir or project_doc.get( 'trainings-dir', None) # if we don't have a trainings dir yet, set to default peek_trainings_dir = peek_trainings_dir or os.path.join( project_dir or "", "trainings") # now, try to load the data.yaml file and see if we have a model definition there data_doc = load_yaml_file(peek_trainings_dir, AI, "data.yaml") if AI else {} # if we do, this overrides everything, also the one from the command line peek_model_name = data_doc.get('model', peek_model_name) # finally, if we have a model name, set up validators if peek_model_name: for fn in Command.find_functions(plugins.get( "vergeml.model", peek_model_name), plugins=plugins): cmd = Command.discover(fn) validators[cmd.name] = ValidateOptions( cmd.options, cmd.name, plugins) except Exception: # in this case we don't care if something went wrong - the error # will be reported later pass # finally, validators for device and data sections validators['device'] = ValidateDevice('device', plugins) validators['data'] = ValidateData('data', plugins) # merge project file if project_file: doc = _load_and_configure(project_file, 'project file', validators) # the project file DOES NOT override values passed to the environment # TODO reserved: hyperparameters and results for k, v in doc.items(): if not k in self._config or self._config[k] is None: self._config[k] = v # after the project file is loaded, fill missing values project_dir = project_dir or '' defaults = { 'samples-dir': os.path.join(project_dir, "samples"), 'test-split': '10%', 'val-split': '10%', 'cache-dir': os.path.join(project_dir, ".cache"), 'random-seed': 2204, 'trainings-dir': os.path.join(project_dir, "trainings"), } for k, v in defaults.items(): if self._config[k] is None: self._config[k] = v # verify split values for split in ('val-split', 'test-split'): spltype, splval = parse_split(self._config[split]) if spltype == 'dir': path = os.path.join(project_dir, splval) if not os.path.exists(path): raise VergeMLError( f"Invalid value for option {split} - no such directory: {splval}", f"Please set {split} to a percentage, number or directory.", hint_key=split, hint_type='value', help_topic='split') self._config[split] = path # need to have data_file variable in outer scope for later when reporting errors data_file = None if self.AI: ai_path = os.path.join(self._config['trainings-dir'], self.AI) if not os.path.exists(ai_path): raise VergeMLError("AI not found: {}".format(self.AI)) # merge data.yaml data_file = os.path.join(self._config['trainings-dir'], self.AI, 'data.yaml') if not os.path.exists(data_file): raise VergeMLError( "data.yaml file not found for AI {}: {}".format( self.AI, data_file)) doc = load_yaml_file(data_file, 'data file') self._config['hyperparameters'] = doc.get('hyperparameters', {}) self._config['results'] = doc.get('results', {}) self._config['model'] = doc.get('model') self.results = _Results(self, data_file) try: # merge device and data config self._config.update(apply_config(config, validators)) except VergeMLError as e: # improve the error message when this runs on the command line if is_global_instance and e.hint_key: key = e.hint_key e.message = f"Option --{key}: " + e.message raise e if self._config['model']: # load the model plugin modelname = self._config['model'] self.model = plugins.get("vergeml.model", modelname) if not self.model: message = f"Unknown model name '{modelname}'" suggestion = did_you_mean(plugins.keys('vergeml.model'), modelname) or "See 'ml help models'." # if model was passed in via --model if model and is_global_instance: message = f"Invalid value for option --model: {message}" else: res = None if not res and data_file: # first check if model was defined in the data file res = _check_definition(data_file, 'model', 'value') if not res and project_file: # next check the project file res = _check_definition(project_file, 'model', 'value') if res: filename, definition = res line, column, length = definition # display a nice error message message = display_err_in_file( filename, line, column, f"{message} {suggestion}", length) # set suggestion to None since it is now contained in message suggestion = None raise VergeMLError(message, suggestion) else: # instantiate the model plugin self.model = self.model(modelname, plugins) # update env from validators for _, plugin in validators.items(): for k, v in plugin.values.items(): self._config[k] = v # always set up numpy and python self.configure('python') self.configure('numpy')
def test_config_invalid(): VALIDATORS = {'device': ValidateDevice()} with pytest.raises(VergeMLError): apply_config({'device': {'id': 'cpu', 'invalid': 'true'}}, VALIDATORS)
def test_config_dict(): VALIDATORS = {'device': ValidateDevice()} res = apply_config({'device': {'id': 'cpu'}}, VALIDATORS) assert (res == {}) assert (VALIDATORS['device'].values['device']['id'] == 'cpu')