def _set_system(self, system):
"""Set system to model.
Create a new system from system dictionary or reconfigure
existing system with system dictionary or copy a system
instance.
Args:
system (dict or system instance): system dictionary
or system instance
Returns:
bool: True if no error occured
"""
if otree.has_base(system, 'System'):
self.system = system
return True
if not isinstance(system, dict): return False
if otree.has_base(self.system, 'System'):
return self.system.set('copy', **system)
self.system = rian.system.new(**system)
return True
def _get_compatibility(self, model):
"""Get compatibility of transformation to given model instance.
Args:
model: rian model instance
Returns:
True if transformation is compatible with given model, else False.
"""
# test type of model instance and subclasses
if not otree.has_base(model, 'Model'):
return False
if not otree.has_base(model.dataset, 'Dataset'):
return False
if not otree.has_base(model.network, 'Network'):
return False
if not otree.has_base(model.system, 'System'):
return False
# check dataset
if (not 'check_dataset' in model.system._default['init']
or model.system._default['init']['check_dataset'] == True) \
and not model.system._check_dataset(model.dataset):
return False
return True
def _set_dataset(self, dataset):
"""Set dataset to model.
Create a new dataset from dataset dictionary or reconfigure
existing dataset with dataset dictionary or copy a dataset
instance.
Args:
dataset (dict or dataset instance): dataset dictionary
or dataset instance
Returns:
bool: True if no error occured
"""
if otree.has_base(dataset, 'Dataset'):
self.dataset = dataset
return True
if not isinstance(dataset, dict): return False
if otree.has_base(self.dataset, 'Dataset'):
return self.dataset.set('copy', **dataset)
self.dataset = rian.dataset.new(**dataset)
return True
def _set_network(self, network):
"""Set network to model.
Create a new network from network dictionary or reconfigure
existing network with network dictionary or copy a network
instance.
Args:
network (dict or network instance): network dictionary
or network instance
Returns:
bool: True if no error occured
"""
if otree.has_base(network, 'Network'):
self.network = network
return True
if not isinstance(network, dict): return False
if otree.has_base(self.network, 'Network'):
return self.network.set('copy', **network)
self.network = rian.network.new(**network)
return True
def test_dataset_import(self):
with self.subTest(filetype="csv"):
dataset = rian.dataset.open('sinus', workspace='testsuite')
test = otree.has_base(dataset, 'Dataset')
self.assertTrue(test)
with self.subTest(filetype="tab"):
dataset = rian.dataset.open('linear', workspace='testsuite')
test = otree.has_base(dataset, 'Dataset')
self.assertTrue(test)
def test_network_create(self) -> None:
with self.subTest(create='autoencoder'):
network = rian.network.create('autoencoder',
columns=['v1', 'v2', 'v3'],
shape=[6, 3, 6])
self.assertTrue(otree.has_base(network, 'Network'))
with self.subTest(create='factor'):
network = rian.network.create('factor',
visible_nodes=['v1', 'v2', 'v3'],
visible_type='gauss',
hidden_nodes=['h1', 'h2'],
hidden_type='sigmoid')
self.assertTrue(otree.has_base(network, 'Network'))
def __init__(self, dataset = None, **kwds):
self.settings = {**self.default, **kwds}
if otree.has_base(dataset, 'Dataset'):
self.settings['dataset'] = dataset
else:
self.settings['dataset'] = rian.dataset.load(dataset)
def save(system,
path=None,
filetype=None,
workspace=None,
base='user',
**kwds):
"""Export system to file.
Args:
system (object): rian system instance
path (str, optional): path of export file
filetype (str, optional): filetype of export file
workspace (str, optional): workspace to use for file export
Returns:
Boolean value which is True if file export was successful
"""
from hup.base import otree
if not otree.has_base(system, 'System'):
raise ValueError("system is not valid")
from hup.base import env
# get directory, filename and fileextension
if isinstance(workspace, str) and not workspace == 'None':
directory = rian.path('systems', workspace=workspace, base=base)
elif isinstance(path, str):
directory = env.get_dirname(path)
else:
directory = env.get_dirname(system.path)
if isinstance(path, str):
name = env.basename(path)
else:
name = system.fullname
if isinstance(filetype, str):
fileext = filetype
elif isinstance(path, str):
fileext = env.fileext(path) or env.fileext(system.path)
else:
fileext = env.fileext(system.path)
path = str(env.join_path(directory, name + '.' + fileext))
# get filetype from file extension if not given
# and test if filetype is supported
if not filetype:
filetype = fileext.lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported.")
# export to file
module_name = filetypes(filetype)[0]
if module_name == 'archive':
return rian.system.exports.archive.save(system, path, filetype, **kwds)
return False
def _check_dataset(self, dataset):
"""Check if dataset contains gauss normalized values."""
if not otree.has_base(dataset, 'Dataset'):
raise TypeError("dataset is not valid")
if not dataset.evaluate('test_gauss'):
raise ValueError("""dataset '%s' is not valid:
GRBMs expect standard normal distributed data.""" %
(dataset.name))
return True
def _check_dataset(self, dataset):
"""Check if dataset contains only binary values."""
if not otree.has_base(dataset, 'Dataset'):
raise ValueError("""could not test dataset:
invalid dataset instance given.""")
if not dataset._get_test_binary():
raise ValueError("""dataset '%s' is not valid:
RBMs expect binary data.""" % dataset.name)
return True
def build(self):
# create model dictionary including dataset, network and system
model_dict = {}
if otree.has_base(self.settings['dataset'], 'Dataset'):
model_dict['dataset'] = self.settings['dataset']
dataset_name = self.settings['dataset'].name
else:
model_dict['dataset'] \
= rian.dataset.load(self.settings['dataset'])
if not model_dict['dataset']: return {}
dataset_name = model_dict['dataset']['config']['name']
if otree.has_base(self.settings['network'], 'Network'):
model_dict['network'] = self.settings['network']
network_name = self.settings['network'].name
else:
model_dict['network'] \
= rian.network.load(self.settings['network'])
if not model_dict['network']: return {}
network_name = model_dict['network']['config']['name']
if otree.has_base(self.settings['system'], 'System'):
model_dict['system'] = self.settings['system']
system_name = self.settings['system'].name
else:
model_dict['system'] \
= rian.system.load(self.settings['system'])
if not model_dict['system']: return {}
system_name = model_dict['system']['config']['name']
model_dict['config'] = {
'name': '-'.join([dataset_name, network_name, system_name]),
'type': 'base.Model'}
# create instance of model
# configure, iniztalize and optimize model
model = rian.model.new(**model_dict)
model.configure()
if self.settings['initialize']: model.initialize()
if self.settings['optimize']: model.optimize()
return model.get('copy')
def test_model_dbn(self) -> None:
with self.subTest(step='create dbn'):
model = rian.model.create(
dataset='linear', network='deep', system='dbn')
self.assertTrue(otree.has_base(model, 'Model'))
with self.subTest(step='optimize dbn'):
model.optimize()
test = model.error < 0.5
self.assertTrue(test)
def initialize(self):
"""Initialize model parameters."""
if not otree.has_base(self.dataset, 'Dataset'):
raise ValueError("dataset is not valid")
if not otree.has_base(self.network, 'Network'):
raise ValueError("network is not valid")
if not otree.has_base(self.system, 'System'):
raise ValueError("system is not valid")
retval = True
# initialize dataset to system including normalization
retval &= bool(self.dataset.initialize(self.system))
# initialize system parameters by using statistics from dataset
retval &= bool(self.system.initialize(self.dataset))
# initialize network parameters with system parameters
retval &= bool(self.network.initialize(self.system))
return retval
def configure(self):
"""Configure model."""
if not otree.has_base(self.dataset, 'Dataset'):
raise ValueError("dataset is not valid")
if not otree.has_base(self.network, 'Network'):
raise ValueError("network is not valid")
if not otree.has_base(self.system, 'System'):
raise ValueError("system is not valid")
retval = True
# configure dataset columns to network
retval &= bool(self.dataset.configure(self.network))
# configure network to restrict nodes to found columns
retval &= bool(self.network.configure(self.dataset))
# configure system to nodes in network
retval &= bool(self.system.configure(self.network))
return retval
def build(self):
# create dataset instance
if not otree.has_base(self.settings['dataset'], 'Dataset'):
return {}
# create network instance from dataset instance
self.settings['network'] = rian.network.create('autoencoder',
dataset = self.settings['dataset'])
return Model(**self.settings).build()
def __init__(self, dataset=None, *args, **kwds):
self.settings = {**self.default, **kwds}
# columns
if 'columns' in self.settings: columns = self.settings['columns']
elif otree.has_base(dataset, 'Dataset'): columns = dataset.columns
else: columns = ['i1', 'i2', 'i3', 'i4', 'o1', 'o2']
self.settings['inputs'] = columns
self.settings['outputs'] = columns
# shape
if 'shape' not in self.settings:
size = len(columns)
self.settings['shape'] = [2 * size, size, 2 * size]
def test_dataset_create(self):
with self.subTest(create="rules"):
dataset = rian.dataset.create(
'rules',
name='example',
columns=['i1', 'i2', 'i3', 'i4', 'o1', 'o2'],
initialize='gauss + bernoulli',
sdev=0.1,
abin=0.5,
rules=[('o1', 'i1 + i2'), ('o2', 'i3 + i4')],
normalize='gauss',
samples=10000)
test = otree.has_base(dataset, 'Dataset')
self.assertTrue(test)
def initialize(self, system=None):
if not system:
return False
if not otree.has_base(system, 'System'):
raise ValueError("system is not valid")
# get edge parameters from system links
for edge in self._graph.edges():
params = system.get('link', edge)
if not params:
continue
edge_dict = self._graph[edge[0]][edge[1]]
edge_dict['params'] = {**edge_dict['params'], **params}
edge_dict['weight'] = float(params['weight'])
return True
def configure(self, dataset=None):
"""Configure network to dataset."""
# check if dataset instance is available
if not otree.has_base(dataset, 'Dataset'):
raise TypeError("dataset is required to be of type dataset")
log.info("configure network: '%s'" % (self._config['name']))
# configure network to dataset
groups = dataset.get('colgroups')
changes = []
for group in groups:
if group not in self._config['nodes'] \
or not (groups[group] == self._config['nodes'][group]):
self._configure_graph(nodelist={
'layer': group,
'list': groups[group]
})
return True
def test_model_import(self) -> None:
with self.subTest(filetype='npz'):
model = rian.model.open('test', workspace='testsuite')
self.assertTrue(otree.has_base(model, 'Model'))
def test_network_import(self) -> None:
with self.subTest(filetype='ini'):
network = rian.network.open('deep', workspace='testsuite')
self.assertTrue(otree.has_base(network, 'Network'))
def save(dataset,
path=None,
filetype=None,
workspace=None,
base='user',
**kwds):
"""Export dataset to file.
Args:
dataset (object): rian dataset instance
path (str, optional): path of export file
filetype (str, optional): filetype of export file
workspace (str, optional): workspace to use for file export
Returns:
Boolean value which is True if file export was successful
"""
if not otree.has_base(dataset, 'Dataset'):
raise TypeError("dataset is not valid")
from hup.base import env
import rian
# get directory, filename and fileextension
if isinstance(workspace, str) and not workspace == 'None':
dname = rian.path('datasets', workspace=workspace, base=base)
elif isinstance(path, str):
dname = env.get_dirname(path)
else:
dname = env.get_dirname(dataset.path)
if isinstance(path, str):
fbase = env.basename(path)
else:
fbase = dataset.fullname
if isinstance(filetype, str):
fext = filetype
elif isinstance(path, str):
fext = env.fileext(path) or env.fileext(dataset.path)
else:
fext = env.fileext(dataset.path)
path = str(env.join_path(dname, fbase + '.' + fext))
# get filetype from file extension if not given
# and test if filetype is supported
if not filetype:
filetype = fext.lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported.")
# export to file
mname = filetypes(filetype)[0]
if mname == 'text':
return text.save(dataset, path, filetype, **kwds)
if mname == 'archive':
return archive.save(dataset, path, filetype, **kwds)
if mname == 'image':
return image.save(dataset, path, filetype, **kwds)
return False
def test_system_import(self) -> None:
with self.subTest(filetype="ini"):
system = rian.system.open('dbn', workspace='testsuite')
test = otree.has_base(system, 'System')
self.assertTrue(test)
