def _set_network(self, network): """Set network to model. Create a new network from network dictionary or reconfigure existing network with network dictionary or copy a network instance. Args: network (dict or network instance): network dictionary or network instance Returns: bool: True if no error occured """ if entity.has_base(network, 'Network'): self.network = network return True if not isinstance(network, dict): return False if entity.has_base(self.network, 'Network'): return self.network.set('copy', **network) self.network = nemoa.network.new(**network) return True
def _set_dataset(self, dataset): """Set dataset to model. Create a new dataset from dataset dictionary or reconfigure existing dataset with dataset dictionary or copy a dataset instance. Args: dataset (dict or dataset instance): dataset dictionary or dataset instance Returns: bool: True if no error occured """ if entity.has_base(dataset, 'Dataset'): self.dataset = dataset return True if not isinstance(dataset, dict): return False if entity.has_base(self.dataset, 'Dataset'): return self.dataset.set('copy', **dataset) self.dataset = nemoa.dataset.new(**dataset) return True
def _set_system(self, system): """Set system to model. Create a new system from system dictionary or reconfigure existing system with system dictionary or copy a system instance. Args: system (dict or system instance): system dictionary or system instance Returns: bool: True if no error occured """ if entity.has_base(system, 'System'): self.system = system return True if not isinstance(system, dict): return False if entity.has_base(self.system, 'System'): return self.system.set('copy', **system) self.system = nemoa.system.new(**system) return True
def _get_compatibility(self, model): """Get compatibility of transformation to given model instance. Args: model: nemoa model instance Returns: True if transformation is compatible with given model, else False. """ # test type of model instance and subclasses if not entity.has_base(model, 'Model'): return False if not entity.has_base(model.dataset, 'Dataset'): return False if not entity.has_base(model.network, 'Network'): return False if not entity.has_base(model.system, 'System'): return False # check dataset if (not 'check_dataset' in model.system._default['init'] or model.system._default['init']['check_dataset'] == True) \ and not model.system._check_dataset(model.dataset): return False return True
def test_dataset_import(self): import nemoa.dataset with self.subTest(filetype="csv"): dataset = nemoa.dataset.open('sinus', workspace='testsuite') test = entity.has_base(dataset, 'Dataset') self.assertTrue(test) with self.subTest(filetype="tab"): dataset = nemoa.dataset.open('linear', workspace='testsuite') test = entity.has_base(dataset, 'Dataset') self.assertTrue(test)
def test_network_create(self): with self.subTest(create='autoencoder'): network = nemoa.network.create('autoencoder', columns=['v1', 'v2', 'v3'], shape=[6, 3, 6]) self.assertTrue(entity.has_base(network, 'Network')) with self.subTest(create='factor'): network = nemoa.network.create('factor', visible_nodes=['v1', 'v2', 'v3'], visible_type='gauss', hidden_nodes=['h1', 'h2'], hidden_type='sigmoid') self.assertTrue(entity.has_base(network, 'Network'))
def __init__(self, dataset=None, **kwds): self.settings = {**self.default, **kwds} if entity.has_base(dataset, 'Dataset'): self.settings['dataset'] = dataset else: self.settings['dataset'] = nemoa.dataset.load(dataset)
def save(system, path=None, filetype=None, workspace=None, base='user', **kwds): """Export system to file. Args: system (object): nemoa system instance path (str, optional): path of export file filetype (str, optional): filetype of export file workspace (str, optional): workspace to use for file export Returns: Boolean value which is True if file export was successful """ from nemoa.base import entity if not entity.has_base(system, 'System'): raise ValueError("system is not valid") from nemoa.base import env # get directory, filename and fileextension if isinstance(workspace, str) and not workspace == 'None': directory = nemoa.path('systems', workspace=workspace, base=base) elif isinstance(path, str): directory = env.get_dirname(path) else: directory = env.get_dirname(system.path) if isinstance(path, str): name = env.basename(path) else: name = system.fullname if isinstance(filetype, str): fileext = filetype elif isinstance(path, str): fileext = env.fileext(path) or env.fileext(system.path) else: fileext = env.fileext(system.path) path = str(env.join_path(directory, name + '.' + fileext)) # get filetype from file extension if not given # and test if filetype is supported if not filetype: filetype = fileext.lower() if filetype not in filetypes(): raise ValueError(f"filetype '{filetype}' is not supported.") # export to file module_name = filetypes(filetype)[0] if module_name == 'archive': return nemoa.system.exports.archive.save(system, path, filetype, **kwds) return False
def save( dataset, path = None, filetype = None, workspace = None, base = 'user', **kwds): """Export dataset to file. Args: dataset (object): nemoa dataset instance path (str, optional): path of export file filetype (str, optional): filetype of export file workspace (str, optional): workspace to use for file export Returns: Boolean value which is True if file export was successful """ if not entity.has_base(dataset, 'Dataset'): raise TypeError("dataset is not valid") from nemoa.base import env import nemoa # get directory, filename and fileextension if isinstance(workspace, str) and not workspace == 'None': dname = nemoa.path('datasets', workspace=workspace, base=base) elif isinstance(path, str): dname = env.get_dirname(path) else: dname = env.get_dirname(dataset.path) if isinstance(path, str): fbase = env.basename(path) else: fbase = dataset.fullname if isinstance(filetype, str): fext = filetype elif isinstance(path, str): fext = env.fileext(path) or env.fileext(dataset.path) else: fext = env.fileext(dataset.path) path = str(env.join_path(dname, fbase + '.' + fext)) # get filetype from file extension if not given # and test if filetype is supported if not filetype: filetype = fext.lower() if filetype not in filetypes(): raise ValueError(f"filetype '{filetype}' is not supported.") # export to file mname = filetypes(filetype)[0] if mname == 'text': return text.save(dataset, path, filetype, **kwds) if mname == 'archive': return archive.save(dataset, path, filetype, **kwds) if mname == 'image': return image.save(dataset, path, filetype, **kwds) return False
def build(self): # create model dictionary including dataset, network and system model_dict = {} if entity.has_base(self.settings['dataset'], 'Dataset'): model_dict['dataset'] = self.settings['dataset'] dataset_name = self.settings['dataset'].name else: model_dict['dataset'] \ = nemoa.dataset.load(self.settings['dataset']) if not model_dict['dataset']: return {} dataset_name = model_dict['dataset']['config']['name'] if entity.has_base(self.settings['network'], 'Network'): model_dict['network'] = self.settings['network'] network_name = self.settings['network'].name else: model_dict['network'] \ = nemoa.network.load(self.settings['network']) if not model_dict['network']: return {} network_name = model_dict['network']['config']['name'] if entity.has_base(self.settings['system'], 'System'): model_dict['system'] = self.settings['system'] system_name = self.settings['system'].name else: model_dict['system'] \ = nemoa.system.load(self.settings['system']) if not model_dict['system']: return {} system_name = model_dict['system']['config']['name'] model_dict['config'] = { 'name': '-'.join([dataset_name, network_name, system_name]), 'type': 'base.Model' } # create instance of model # configure, iniztalize and optimize model model = nemoa.model.new(**model_dict) model.configure() if self.settings['initialize']: model.initialize() if self.settings['optimize']: model.optimize() return model.get('copy')
def _check_dataset(self, dataset): """Check if dataset contains gauss normalized values.""" if not entity.has_base(dataset, 'Dataset'): raise TypeError("dataset is not valid") if not dataset.evaluate('test_gauss'): raise ValueError("""dataset '%s' is not valid: GRBMs expect standard normal distributed data.""" % (dataset.name)) return True
def _check_dataset(self, dataset): """Check if dataset contains only binary values.""" if not entity.has_base(dataset, 'Dataset'): raise ValueError("""could not test dataset: invalid dataset instance given.""") if not dataset._get_test_binary(): raise ValueError("""dataset '%s' is not valid: RBMs expect binary data.""" % dataset.name) return True
def configure(self): """Configure model.""" if not entity.has_base(self.dataset, 'Dataset'): raise ValueError("dataset is not valid") if not entity.has_base(self.network, 'Network'): raise ValueError("network is not valid") if not entity.has_base(self.system, 'System'): raise ValueError("system is not valid") retval = True # configure dataset columns to network retval &= bool(self.dataset.configure(self.network)) # configure network to restrict nodes to found columns retval &= bool(self.network.configure(self.dataset)) # configure system to nodes in network retval &= bool(self.system.configure(self.network)) return retval
def initialize(self): """Initialize model parameters.""" if not entity.has_base(self.dataset, 'Dataset'): raise ValueError("dataset is not valid") if not entity.has_base(self.network, 'Network'): raise ValueError("network is not valid") if not entity.has_base(self.system, 'System'): raise ValueError("system is not valid") retval = True # initialize dataset to system including normalization retval &= bool(self.dataset.initialize(self.system)) # initialize system parameters by using statistics from dataset retval &= bool(self.system.initialize(self.dataset)) # initialize network parameters with system parameters retval &= bool(self.network.initialize(self.system)) return retval
def build(self): # create dataset instance if not entity.has_base(self.settings['dataset'], 'Dataset'): return {} # create network instance from dataset instance self.settings['network'] = nemoa.network.create( 'autoencoder', dataset=self.settings['dataset']) return Model(**self.settings).build()
def test_model_ann(self): with self.subTest(step='create shallow ann'): model = nemoa.model.create(dataset='linear', network='shallow', system='ann') self.assertTrue(entity.has_base(model, 'Model')) with self.subTest(step='optimize shallow ann'): model.optimize() test = model.error < 0.1 self.assertTrue(test)
def test_model_dbn(self): with self.subTest(step='create dbn'): model = nemoa.model.create(dataset='linear', network='deep', system='dbn') self.assertTrue(entity.has_base(model, 'Model')) with self.subTest(step='optimize dbn'): model.optimize() test = model.error < 0.5 self.assertTrue(test)
def test_dataset_create(self): import nemoa.dataset with self.subTest(create = "rules"): dataset = nemoa.dataset.create('rules', name = 'example', columns = ['i1', 'i2', 'i3', 'i4', 'o1', 'o2'], initialize = 'gauss + bernoulli', sdev = 0.1, abin = 0.5, rules = [('o1', 'i1 + i2'), ('o2', 'i3 + i4')], normalize = 'gauss', samples = 10000) test = entity.has_base(dataset, 'Dataset') self.assertTrue(test)
def __init__(self, dataset=None, *args, **kwds): self.settings = {**self.default, **kwds} # columns if 'columns' in self.settings: columns = self.settings['columns'] elif entity.has_base(dataset, 'Dataset'): columns = dataset.columns else: columns = ['i1', 'i2', 'i3', 'i4', 'o1', 'o2'] self.settings['inputs'] = columns self.settings['outputs'] = columns # shape if 'shape' not in self.settings: size = len(columns) self.settings['shape'] = [2 * size, size, 2 * size]
def initialize(self, system=None): if not system: return False if not entity.has_base(system, 'System'): raise ValueError("system is not valid") # get edge parameters from system links for edge in self._graph.edges(): params = system.get('link', edge) if not params: continue edge_dict = self._graph[edge[0]][edge[1]] edge_dict['params'] = {**edge_dict['params'], **params} edge_dict['weight'] = float(params['weight']) return True
def configure(self, dataset=None): """Configure network to dataset.""" # check if dataset instance is available if not entity.has_base(dataset, 'Dataset'): raise TypeError("dataset is required to be of type dataset") log.info("configure network: '%s'" % (self._config['name'])) # configure network to dataset groups = dataset.get('colgroups') changes = [] for group in groups: if group not in self._config['nodes'] \ or not (groups[group] == self._config['nodes'][group]): self._configure_graph(nodelist={ 'layer': group, 'list': groups[group] }) return True
def test_system_import(self): with self.subTest(filetype="ini"): system = nemoa.system.open('dbn', workspace='testsuite') test = entity.has_base(system, 'System') self.assertTrue(test)
def test_model_import(self): with self.subTest(filetype='npz'): model = nemoa.model.open('test', workspace='testsuite') self.assertTrue(entity.has_base(model, 'Model'))
def test_network_import(self): with self.subTest(filetype='ini'): network = nemoa.network.open('deep', workspace='testsuite') self.assertTrue(entity.has_base(network, 'Network'))
def save(dataset, path=None, filetype=None, workspace=None, base='user', **kwds): """Export dataset to file. Args: dataset (object): nemoa dataset instance path (str, optional): path of export file filetype (str, optional): filetype of export file workspace (str, optional): workspace to use for file export Returns: Boolean value which is True if file export was successful """ if not entity.has_base(dataset, 'Dataset'): raise TypeError("dataset is not valid") from nemoa.base import env import nemoa # get directory, filename and fileextension if isinstance(workspace, str) and not workspace == 'None': dname = nemoa.path('datasets', workspace=workspace, base=base) elif isinstance(path, str): dname = env.get_dirname(path) else: dname = env.get_dirname(dataset.path) if isinstance(path, str): fbase = env.basename(path) else: fbase = dataset.fullname if isinstance(filetype, str): fext = filetype elif isinstance(path, str): fext = env.fileext(path) or env.fileext(dataset.path) else: fext = env.fileext(dataset.path) path = str(env.join_path(dname, fbase + '.' + fext)) # get filetype from file extension if not given # and test if filetype is supported if not filetype: filetype = fext.lower() if filetype not in filetypes(): raise ValueError(f"filetype '{filetype}' is not supported.") # export to file mname = filetypes(filetype)[0] if mname == 'text': return text.save(dataset, path, filetype, **kwds) if mname == 'archive': return archive.save(dataset, path, filetype, **kwds) if mname == 'image': return image.save(dataset, path, filetype, **kwds) return False