def save(system,
path=None,
filetype=None,
workspace=None,
base='user',
**kwds):
"""Export system to file.
Args:
system (object): nemoa system instance
path (str, optional): path of export file
filetype (str, optional): filetype of export file
workspace (str, optional): workspace to use for file export
Returns:
Boolean value which is True if file export was successful
"""
from nemoa.base import entity
if not entity.has_base(system, 'System'):
raise ValueError("system is not valid")
from nemoa.base import env
# get directory, filename and fileextension
if isinstance(workspace, str) and not workspace == 'None':
directory = nemoa.path('systems', workspace=workspace, base=base)
elif isinstance(path, str):
directory = env.get_dirname(path)
else:
directory = env.get_dirname(system.path)
if isinstance(path, str):
name = env.basename(path)
else:
name = system.fullname
if isinstance(filetype, str):
fileext = filetype
elif isinstance(path, str):
fileext = env.fileext(path) or env.fileext(system.path)
else:
fileext = env.fileext(system.path)
path = str(env.join_path(directory, name + '.' + fileext))
# get filetype from file extension if not given
# and test if filetype is supported
if not filetype:
filetype = fileext.lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported.")
# export to file
module_name = filetypes(filetype)[0]
if module_name == 'archive':
return nemoa.system.exports.archive.save(system, path, filetype,
**kwds)
return False
def save(
dataset, path = None, filetype = None, workspace = None,
base = 'user', **kwds):
"""Export dataset to file.
Args:
dataset (object): nemoa dataset instance
path (str, optional): path of export file
filetype (str, optional): filetype of export file
workspace (str, optional): workspace to use for file export
Returns:
Boolean value which is True if file export was successful
"""
if not entity.has_base(dataset, 'Dataset'):
raise TypeError("dataset is not valid")
from nemoa.base import env
import nemoa
# get directory, filename and fileextension
if isinstance(workspace, str) and not workspace == 'None':
dname = nemoa.path('datasets', workspace=workspace, base=base)
elif isinstance(path, str):
dname = env.get_dirname(path)
else:
dname = env.get_dirname(dataset.path)
if isinstance(path, str):
fbase = env.basename(path)
else:
fbase = dataset.fullname
if isinstance(filetype, str):
fext = filetype
elif isinstance(path, str):
fext = env.fileext(path) or env.fileext(dataset.path)
else:
fext = env.fileext(dataset.path)
path = str(env.join_path(dname, fbase + '.' + fext))
# get filetype from file extension if not given
# and test if filetype is supported
if not filetype:
filetype = fext.lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported.")
# export to file
mname = filetypes(filetype)[0]
if mname == 'text':
return text.save(dataset, path, filetype, **kwds)
if mname == 'archive':
return archive.save(dataset, path, filetype, **kwds)
if mname == 'image':
return image.save(dataset, path, filetype, **kwds)
return False
def load(path, **kwds):
"""Import network from text file."""
from nemoa.base import env
# extract filetype from path
filetype = env.fileext(path).lower()
# test if filetype is supported
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported")
if filetype in ['ini', 'txt']: return Ini(**kwds).load(path)
return False
def load(path, **kwds):
"""Import dataset from text file."""
from nemoa.base import env
# get extract filetype from file extension
filetype = env.fileext(path).lower()
# test if filetype is supported
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported")
if filetype == 'csv': return Csv(**kwds).load(path)
if filetype in ['tsv', 'tab']: return Tsv(**kwds).load(path)
return False
def load(path, **kwds):
"""Import network from graph description file."""
from nemoa.base import env
# extract filetype from path
filetype = env.fileext(path).lower()
# test if filetype is supported
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported")
if filetype == 'gml': return Gml(**kwds).load(path)
if filetype in ['graphml', 'xml']: return Graphml(**kwds).load(path)
if filetype == 'dot': return Dot(**kwds).load(path)
return False
def load(path, **kwds):
"""Import dataset from text file."""
from nemoa.base import env
# get extract filetype from file extension
filetype = env.fileext(path).lower()
# test if filetype is supported
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported")
if filetype == 'csv': return Csv(**kwds).load(path)
if filetype in ['tsv', 'tab']: return Tsv(**kwds).load(path)
return False
def load(path, filetype=None, **kwds):
"""Import network dictionary from file or workspace."""
import os
from nemoa.base import env
import nemoa
# get path (if necessary)
if 'workspace' in kwds or not os.path.isfile(path):
name = path
pathkwds = {}
if 'workspace' in kwds:
pathkwds['workspace'] = kwds.pop('workspace')
if 'base' in kwds:
pathkwds['base'] = kwds.pop('base')
path = nemoa.path('network', name, **pathkwds)
if not path:
raise Warning("""could not import network:
invalid network name.""") or {}
if not os.path.isfile(path):
raise Warning("""could not import network:
file '%s' does not exist.""" % path) or {}
# get filetype (from file extension if not given)
# and check if filetype is supported
if not filetype: filetype = env.fileext(path).lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported")
# import, check and update dictionary
mname = filetypes(filetype)[0]
if mname == 'archive':
network = archive.load(path, **kwds)
elif mname == 'graph':
network = graph.load(path, **kwds)
elif mname == 'text':
network = text.load(path, **kwds)
else:
network = None
if not network:
raise ValueError("""could not import network:
file '%s' is not valid.""" % path) or {}
# update path
network['config']['path'] = path
return network
def load(path, filetype=None, **kwds):
"""Import system dictionary from file or workspace."""
import os
from nemoa.base import env
# get path (if necessary)
if 'workspace' in kwds or not os.path.isfile(path):
name = path
pathkwds = {}
if 'workspace' in kwds:
pathkwds['workspace'] = kwds.pop('workspace')
if 'base' in kwds:
pathkwds['base'] = kwds.pop('base')
path = nemoa.path('system', name, **pathkwds)
if not path:
raise Warning("""could not import system:
invalid system name.""") or {}
if not os.path.isfile(path):
raise Warning("""could not import system:
file '%s' does not exist.""" % path) or {}
# get filtype from file extension if not given
# and check if filetype is supported
if not filetype:
filetype = env.fileext(path).lower()
if filetype not in filetypes():
raise ValueError("""could not import system:
filetype '%s' is not supported.""" % filetype)
# import and check dictionary
mname = filetypes(filetype)[0]
if mname == 'archive':
system = nemoa.system.imports.archive.load(path, **kwds)
elif mname == 'text':
system = nemoa.system.imports.text.load(path, **kwds)
else:
system = None
if not system:
raise ValueError("""could not import system:
file '%s' is not valid.""" % path) or {}
# update path
system['config']['path'] = path
return system
def load(path, **kwds):
"""Import system from text file."""
from nemoa.base import env
# extract filetype from path
filetype = env.fileext(path).lower()
# test if filetype is supported
if filetype not in filetypes():
raise ValueError("""could not import graph:
filetype '%s' is not supported.""" % (filetype))
if filetype in ['ini', 'txt']:
return Ini(**kwds).load(path)
return False
def load(path, filetype=None, **kwds):
"""Import model dictionary from file or workspace."""
import os
from nemoa.base import env
# get path (if necessary)
if 'workspace' in kwds or not os.path.isfile(path):
name = path
pathkwds = {}
if 'workspace' in kwds:
pathkwds['workspace'] = kwds.pop('workspace')
if 'base' in kwds:
pathkwds['base'] = kwds.pop('base')
path = nemoa.path('model', name, **pathkwds)
#path = nemoa.session.get('path', 'model', name, **pathkwds)
if not path:
raise Warning(f"unknown model '{name}'")
if not os.path.isfile(path):
raise IOError(f"file '{path}' does not exist")
# get filtype from file extension if not given
# and check if filetype is supported
if not filetype: filetype = env.fileext(path).lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported")
# import and check dictionary
mname = filetypes(filetype)[0]
if mname == 'archive':
model = nemoa.model.imports.archive.load(path, **kwds)
else:
model = None
if not model:
raise ValueError(f"file '{path}' is not valid")
# update path
model['config']['path'] = path
return model
def load(path, filetype=None, **kwds):
"""Import dataset dictionary from file or workspace."""
import os
from nemoa.base import env
# get path (if necessary)
if 'workspace' in kwds or not os.path.isfile(path):
name = path
pathkwds = {}
if 'workspace' in kwds:
pathkwds['workspace'] = kwds.pop('workspace')
if 'base' in kwds:
pathkwds['base'] = kwds.pop('base')
path = nemoa.path('dataset', name, **pathkwds)
if not path:
raise Warning("""could not import dataset:
invalid dataset name.""") or {}
if not os.path.isfile(path):
raise Warning("""could not import dataset:
file '%s' does not exist.""" % path) or {}
# get filtype from file extension if not given
# and check if filetype is supported
if not filetype: filetype = env.fileext(path).lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported")
# import and check dictionary
mname = filetypes(filetype)[0]
if mname == 'archive': dataset = archive.load(path, **kwds)
elif mname == 'text': dataset = text.load(path, **kwds)
else: dataset = None
if not dataset:
raise ValueError("""could not import dataset:
file '%s' is not valid.""" % path) or {}
# update path
dataset['config']['path'] = path
return dataset
def load(arg, base=None, filetype=None, **kwds):
"""Import workspace dictionary from file or workspace."""
import os
from nemoa.base import env
if os.path.isfile(arg):
path = arg
workspace = os.path.basename(os.path.dirname(path))
base = None # 2Do: get base and workspace from path
else:
path = nemoa.path('inifile', workspace=arg, base=base)
if not path:
return None
else:
workspace = arg
# get filtype from file extension if not given
# and check if filetype is supported
if not filetype: filetype = env.fileext(path).lower()
if filetype not in filetypes():
raise ValueError("""could not import workspace:
filetype '%s' is not supported.""" % filetype)
# import and check dictionary
mname = filetypes(filetype)[0]
if mname == 'text':
config = nemoa.workspace.imports.text.load(path, **kwds)
else:
config = None
if not config:
raise ValueError("""could not import workspace:
file '%s' is not valid.""" % path) or {}
# update path
config['config']['path'] = path
config['config']['name'] = workspace
config['config']['base'] = base
return config
def test_fileext(self) -> None:
path = (('a', ('b', 'c')), 'd', 'base.ext')
val = env.fileext(*path)
self.assertEqual(val, 'ext')
def test_fileext(self) -> None:
path = (('a', ('b', 'c')), 'd', 'base.ext')
val = env.fileext(*path)
self.assertEqual(val, 'ext')
def save(dataset,
path=None,
filetype=None,
workspace=None,
base='user',
**kwds):
"""Export dataset to file.
Args:
dataset (object): nemoa dataset instance
path (str, optional): path of export file
filetype (str, optional): filetype of export file
workspace (str, optional): workspace to use for file export
Returns:
Boolean value which is True if file export was successful
"""
if not entity.has_base(dataset, 'Dataset'):
raise TypeError("dataset is not valid")
from nemoa.base import env
import nemoa
# get directory, filename and fileextension
if isinstance(workspace, str) and not workspace == 'None':
dname = nemoa.path('datasets', workspace=workspace, base=base)
elif isinstance(path, str):
dname = env.get_dirname(path)
else:
dname = env.get_dirname(dataset.path)
if isinstance(path, str):
fbase = env.basename(path)
else:
fbase = dataset.fullname
if isinstance(filetype, str):
fext = filetype
elif isinstance(path, str):
fext = env.fileext(path) or env.fileext(dataset.path)
else:
fext = env.fileext(dataset.path)
path = str(env.join_path(dname, fbase + '.' + fext))
# get filetype from file extension if not given
# and test if filetype is supported
if not filetype:
filetype = fext.lower()
if filetype not in filetypes():
raise ValueError(f"filetype '{filetype}' is not supported.")
# export to file
mname = filetypes(filetype)[0]
if mname == 'text':
return text.save(dataset, path, filetype, **kwds)
if mname == 'archive':
return archive.save(dataset, path, filetype, **kwds)
if mname == 'image':
return image.save(dataset, path, filetype, **kwds)
return False
def _set_workspace_scandir(self, *args, **kwds):
"""Scan workspace for files."""
import glob
from nemoa.base import env
# change current base and workspace (if necessary)
cur_workspace = self._get_workspace()
cur_base = self._get_base()
if args:
workspace = args[0]
else:
workspace = cur_workspace
if 'base' in kwds:
base = kwds.pop('base')
else:
base = cur_base
if (base, workspace) != (cur_base, cur_workspace):
current = self._config.get('workspace', None)
chdir = self._set_workspace(workspace, base=base)
else:
chdir = False
# scan workspace for objects
for objtype in list(self._config['register'].keys()):
dirmask = self._config['current']['path'][objtype + 's']
filemask = self._get_path_expand(dirmask, '*.*')
if objtype == 'dataset':
from nemoa.dataset import imports
filetypes = imports.filetypes()
elif objtype == 'network':
from nemoa.network import imports
filetypes = imports.filetypes()
elif objtype == 'system':
from nemoa.system import imports
filetypes = imports.filetypes()
elif objtype == 'model':
from nemoa.model import imports
filetypes = imports.filetypes()
elif objtype == 'script':
filetypes = ['py']
# scan for files
objregister = self._config['register'][objtype]
for filepath in glob.iglob(filemask):
filetype = env.fileext(filepath)
if filetype not in filetypes:
continue
basename = env.basename(filepath)
filespace = self._get_workspace()
filebase = self._get_base()
fullname = '%s.%s.%s' % (filebase, filespace, basename)
if fullname in objregister:
continue
# register object configuration
objregister[fullname] = {
'base': filebase,
'fullname': fullname,
'name': basename,
'path': filepath,
'type': objtype,
'workspace': filespace
}
# change to previous workspace if necessary
if chdir:
if current:
self._set_workspace(current.get('name'),
base=current.get('base'))
else:
self._set_workspace(None)
return True
