def save(system,
path=None,
filetype=None,
workspace=None,
base='user',
**kwds):
"""Export system to file.
Args:
system (object): nemoa system instance
path (str, optional): path of export file
filetype (str, optional): filetype of export file
workspace (str, optional): workspace to use for file export
Returns:
Boolean value which is True if file export was successful
"""
from nemoa.base import entity
if not entity.has_base(system, 'System'):
raise ValueError("system is not valid")
from nemoa.base import env
# get directory, filename and fileextension
if isinstance(workspace, str) and not workspace == 'None':
directory = nemoa.path('systems', workspace=workspace, base=base)
elif isinstance(path, str):
directory = env.get_dirname(path)
else:
directory = env.get_dirname(system.path)
if isinstance(path, str):
name = env.basename(path)
else:
name = system.fullname
if isinstance(filetype, str):
fileext = filetype
elif isinstance(path, str):
fileext = env.fileext(path) or env.fileext(system.path)
else:
fileext = env.fileext(system.path)
path = str(env.join_path(directory, name + '.' + fileext))
# get filetype from file extension if not given
# and test if filetype is supported
if not filetype:
filetype = fileext.lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported.")
# export to file
module_name = filetypes(filetype)[0]
if module_name == 'archive':
return nemoa.system.exports.archive.save(system, path, filetype,
**kwds)
return False
def save(
dataset, path = None, filetype = None, workspace = None,
base = 'user', **kwds):
"""Export dataset to file.
Args:
dataset (object): nemoa dataset instance
path (str, optional): path of export file
filetype (str, optional): filetype of export file
workspace (str, optional): workspace to use for file export
Returns:
Boolean value which is True if file export was successful
"""
if not entity.has_base(dataset, 'Dataset'):
raise TypeError("dataset is not valid")
from nemoa.base import env
import nemoa
# get directory, filename and fileextension
if isinstance(workspace, str) and not workspace == 'None':
dname = nemoa.path('datasets', workspace=workspace, base=base)
elif isinstance(path, str):
dname = env.get_dirname(path)
else:
dname = env.get_dirname(dataset.path)
if isinstance(path, str):
fbase = env.basename(path)
else:
fbase = dataset.fullname
if isinstance(filetype, str):
fext = filetype
elif isinstance(path, str):
fext = env.fileext(path) or env.fileext(dataset.path)
else:
fext = env.fileext(dataset.path)
path = str(env.join_path(dname, fbase + '.' + fext))
# get filetype from file extension if not given
# and test if filetype is supported
if not filetype:
filetype = fext.lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported.")
# export to file
mname = filetypes(filetype)[0]
if mname == 'text':
return text.save(dataset, path, filetype, **kwds)
if mname == 'archive':
return archive.save(dataset, path, filetype, **kwds)
if mname == 'image':
return image.save(dataset, path, filetype, **kwds)
return False
def load(self, path):
"""Return system configuration as dictionary.
Args:
path: configuration file used to generate system
configuration dictionary.
"""
from nemoa.base import env
from nemoa.file import inifile
# import ini file to dictionary, using ini file structure
# described with regular expressions
system = inifile.load(
path, {
'system': {
'name': str,
'type': str
},
'schedule [.0-9a-zA-Z]*': {
'system [.0-9a-zA-Z]*': dict
}
})
if not system \
or not 'system' in system \
or not 'type' in system['system']:
raise ValueError("""could not import system:
configuration file '%s' is not valid.""" % (path))
config = system['system'].copy()
# update / set name
if 'name' not in config: config['name'] = env.basename(path)
# update / set optimization schedules
schedules = {}
for key in system:
if key[:8].lower() != 'schedule': continue
name = key[9:]
schedules[name] = {'name': name}
for syskey in system[key]:
if syskey[:6].lower() != 'system': continue
systype = syskey[7:]
schedules[name][systype] = system[key][syskey].copy()
config['schedules'] = schedules
return {'config': config}
def test_basename(self) -> None:
path = (('a', ('b', 'c')), 'd', 'base.ext')
val = env.basename(*path)
self.assertEqual(val, 'base')
def load(self, path):
"""Get dataset configuration and dataset tables.
Args:
path (string): csv file containing dataset configuration and
dataset table.
"""
from nemoa.file import csvfile, inifile
from nemoa.base import env
# get CSVFile instance
file = csvfile.CSVFile(path)
# Get configuration from CSV comment lines
comment = file.comment
structure = {
'name': str,
'branch': str,
'version': int,
'about': str,
'author': str,
'email': str,
'license': str,
'filetype': str,
'application': str,
'preprocessing': dict,
'type': str,
'labelformat': str
}
config = inifile.decode(comment, flat=True, structure=structure)
if 'name' in config:
name = config['name']
else:
name = env.basename(path)
config['name'] = name
if 'type' not in config:
config['type'] = 'base.Dataset'
# add column and row filters
config['colfilter'] = {'*': ['*:*']}
config['rowfilter'] = {'*': ['*:*'], name: [name + ':*']}
data = file.select()
config['table'] = {name: config.copy()}
config['table'][name]['fraction'] = 1.0
config['columns'] = tuple()
config['colmapping'] = {}
config['table'][name]['columns'] = []
for column in data.dtype.names:
if column == 'label': continue
config['columns'] += (('', column), )
config['colmapping'][column] = column
config['table'][name]['columns'].append(column)
# get data table from csv data
tables = {name: data}
return {'config': config, 'tables': tables}
def _parse_layer_network(self, path):
from nemoa.base import env
from nemoa.file import inifile
structure = {
'network': {
'name': str,
'description': str,
'branch': str,
'version': str,
'about': str,
'author': str,
'email': str,
'license': str,
'type': str,
'layers': list,
'directed': bool,
'labelformat': str
},
'layer [0-9a-zA-Z]*': {
'name': str,
'function': str,
'distribution': str,
'type': str,
'visible': bool,
'file': str,
'nodes': list,
'size': int
},
'binding [0-9a-zA-Z]*-[0-9a-zA-Z]*': {
'[0-9a-zA-Z]*': list
}
}
ini_dict = inifile.load(path, structure=structure)
config = ini_dict['network'].copy()
# layers
if 'layers' not in config:
raise Warning("""file '%s' does not
contain parameter 'layers'.""" % path)
config['layer'] = config['layers']
del config['layers']
# name
if 'name' not in config: config['name'] = env.basename(path)
# directed
if 'directed' not in config: config['directed'] = True
# node labelformat
if 'labelformat' not in config:
config['labelformat'] = 'generic:string'
# init network dictionary
config['nodes'] = {}
config['edges'] = {}
config['layers'] = {}
visible_layer_ids = [0, len(config['layer']) - 1]
# [layer *]
for layer_id, layer in enumerate(config['layer']):
layer_section = 'layer ' + layer
if layer_section not in ini_dict:
raise Warning("""could not import layer
network: file '%s' does not contain information
about layer '%s'.""" % (path, layer))
sec_data = ini_dict[layer_section]
config['layers'][layer] = {}
lay_data = config['layers'][layer]
# get name of layer
lay_data['name'] = sec_data.get('name')
# get visibility (observable, latent) of nodes in layer
if 'visible' in sec_data:
lay_data['visible'] = sec_data['visible']
else:
lay_data['visible'] = layer_id in visible_layer_ids
# get type of layer nodes
if 'type' in sec_data:
lay_data['type'] = sec_data.get('type')
# get function of layer nodes
if 'function' in sec_data:
lay_data['function'] = sec_data.get('function')
# get distribution of layer nodes
if 'distribution' in sec_data:
lay_data['distribution'] = sec_data.get('distribution')
# get nodes of layer from given list file ('file')
# or from list ('nodes') or from given layer size ('size')
if 'file' in sec_data:
file_str = sec_data['file']
list_file = nemoa.workspace._expand_path(file_str)
if not os.path.exists(list_file):
raise ValueError("""listfile '%s'
does not exists!""" % list_file)
with open(list_file, 'r') as list_file:
fileLines = list_file.readlines()
node_list = [node.strip() for node in fileLines]
elif 'nodes' in sec_data:
node_list = sec_data['nodes']
elif 'size' in sec_data:
node_list = ['n%s' % (n) \
for n in range(1, sec_data['size'] + 1)]
else:
raise Warning("""could not import layer
network: layer '%s' does not contain valid node
information!""" % (layer))
config['nodes'][layer] = []
for node in node_list:
node = node.strip()
if node == '': continue
if node not in config['nodes'][layer]:
config['nodes'][layer].append(node)
# parse '[binding *]' sections and add edges to network dict
for i in range(len(config['layer']) - 1):
src_layer = config['layer'][i]
tgt_layer = config['layer'][i + 1]
edge_layer = (src_layer, tgt_layer)
config['edges'][edge_layer] = []
edge_section = 'binding %s-%s' % (src_layer, tgt_layer)
# create full binding between two layers if not specified
if edge_section not in ini_dict:
for src_node in config['nodes'][src_layer]:
for tgt_node in config['nodes'][tgt_layer]:
edge = (src_node, tgt_node)
config['edges'][edge_layer].append(edge)
continue
# get edges from '[binding *]' section
for src_node in ini_dict[edge_section]:
src_node = src_node.strip()
if src_node == '': continue
if src_node not in config['nodes'][src_layer]: continue
for tgt_node in ini_dict[edge_section][src_node]:
tgt_node = tgt_node.strip()
if tgt_node == '': continue
if tgt_node not in config['nodes'][tgt_layer]: continue
edge = (src_node, tgt_node)
if edge in config['edges'][edge_layer]: continue
config['edges'][edge_layer].append(edge)
# check network binding
for i in range(len(config['layer']) - 1):
src_layer = config['layer'][i]
tgt_layer = config['layer'][i + 1]
edge_layer = (src_layer, tgt_layer)
if config['edges'][edge_layer] == []:
raise Warning("""layer '%s' and
layer '%s' are not connected!""" % (src_layer, tgt_layer))
return {'config': config}
def test_basename(self) -> None:
path = (('a', ('b', 'c')), 'd', 'base.ext')
val = env.basename(*path)
self.assertEqual(val, 'base')
def save(dataset,
path=None,
filetype=None,
workspace=None,
base='user',
**kwds):
"""Export dataset to file.
Args:
dataset (object): nemoa dataset instance
path (str, optional): path of export file
filetype (str, optional): filetype of export file
workspace (str, optional): workspace to use for file export
Returns:
Boolean value which is True if file export was successful
"""
if not entity.has_base(dataset, 'Dataset'):
raise TypeError("dataset is not valid")
from nemoa.base import env
import nemoa
# get directory, filename and fileextension
if isinstance(workspace, str) and not workspace == 'None':
dname = nemoa.path('datasets', workspace=workspace, base=base)
elif isinstance(path, str):
dname = env.get_dirname(path)
else:
dname = env.get_dirname(dataset.path)
if isinstance(path, str):
fbase = env.basename(path)
else:
fbase = dataset.fullname
if isinstance(filetype, str):
fext = filetype
elif isinstance(path, str):
fext = env.fileext(path) or env.fileext(dataset.path)
else:
fext = env.fileext(dataset.path)
path = str(env.join_path(dname, fbase + '.' + fext))
# get filetype from file extension if not given
# and test if filetype is supported
if not filetype:
filetype = fext.lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported.")
# export to file
mname = filetypes(filetype)[0]
if mname == 'text':
return text.save(dataset, path, filetype, **kwds)
if mname == 'archive':
return archive.save(dataset, path, filetype, **kwds)
if mname == 'image':
return image.save(dataset, path, filetype, **kwds)
return False
def _set_workspace_scandir(self, *args, **kwds):
"""Scan workspace for files."""
import glob
from nemoa.base import env
# change current base and workspace (if necessary)
cur_workspace = self._get_workspace()
cur_base = self._get_base()
if args:
workspace = args[0]
else:
workspace = cur_workspace
if 'base' in kwds:
base = kwds.pop('base')
else:
base = cur_base
if (base, workspace) != (cur_base, cur_workspace):
current = self._config.get('workspace', None)
chdir = self._set_workspace(workspace, base=base)
else:
chdir = False
# scan workspace for objects
for objtype in list(self._config['register'].keys()):
dirmask = self._config['current']['path'][objtype + 's']
filemask = self._get_path_expand(dirmask, '*.*')
if objtype == 'dataset':
from nemoa.dataset import imports
filetypes = imports.filetypes()
elif objtype == 'network':
from nemoa.network import imports
filetypes = imports.filetypes()
elif objtype == 'system':
from nemoa.system import imports
filetypes = imports.filetypes()
elif objtype == 'model':
from nemoa.model import imports
filetypes = imports.filetypes()
elif objtype == 'script':
filetypes = ['py']
# scan for files
objregister = self._config['register'][objtype]
for filepath in glob.iglob(filemask):
filetype = env.fileext(filepath)
if filetype not in filetypes:
continue
basename = env.basename(filepath)
filespace = self._get_workspace()
filebase = self._get_base()
fullname = '%s.%s.%s' % (filebase, filespace, basename)
if fullname in objregister:
continue
# register object configuration
objregister[fullname] = {
'base': filebase,
'fullname': fullname,
'name': basename,
'path': filepath,
'type': objtype,
'workspace': filespace
}
# change to previous workspace if necessary
if chdir:
if current:
self._set_workspace(current.get('name'),
base=current.get('base'))
else:
self._set_workspace(None)
return True
def load(self, path):
"""Get dataset configuration and dataset tables.
Args:
path (string): csv file containing dataset configuration and
dataset table.
"""
from nemoa.file import csvfile, inifile
from nemoa.base import env
# get CSVFile instance
file = csvfile.CSVFile(path)
# Get configuration from CSV comment lines
comment = file.comment
structure = {
'name': str,
'branch': str,
'version': int,
'about': str,
'author': str,
'email': str,
'license': str,
'filetype': str,
'application': str,
'preprocessing': dict,
'type': str,
'labelformat': str}
config = inifile.decode(comment, flat=True, structure=structure)
if 'name' in config:
name = config['name']
else:
name = env.basename(path)
config['name'] = name
if 'type' not in config:
config['type'] = 'base.Dataset'
# add column and row filters
config['colfilter'] = {'*': ['*:*']}
config['rowfilter'] = {'*': ['*:*'], name: [name + ':*']}
data = file.select()
config['table'] = {name: config.copy()}
config['table'][name]['fraction'] = 1.0
config['columns'] = tuple()
config['colmapping'] = {}
config['table'][name]['columns'] = []
for column in data.dtype.names:
if column == 'label': continue
config['columns'] += (('', column),)
config['colmapping'][column] = column
config['table'][name]['columns'].append(column)
# get data table from csv data
tables = { name: data }
return { 'config': config, 'tables': tables }