Exemplo n.º 1
0
    def __importModelFromFile(self, file):
        """Return new model instance and set configuration and parameters from file."""

        nemoa.log('import model from file')
        nemoa.setLog(indent='+1')

        # check file
        if not os.path.exists(file):
            if os.path.exists(nemoa.workspace.path('models') + file + '.mp'):
                file = nemoa.workspace.path('models') + file + '.mp'
            else:
                return nemoa.log(
                    'error', """
                could not load model '%s':
                file does not exist.""" % file)

        # load model parameters and configuration from file
        nemoa.log('load model: \'%s\'' % file)
        modelDict = nemoa.common.dictFromFile(file)

        model = self.__getModelInstance(name=modelDict['config']['name'],
                                        config=modelDict['config'],
                                        dataset=modelDict['dataset']['cfg'],
                                        network=modelDict['network']['cfg'],
                                        system=modelDict['system']['config'])

        if nemoa.type.isModel(model): model._set(modelDict)
        else: return None

        nemoa.setLog(indent='-1')
        return model
Exemplo n.º 2
0
    def loadProject(self, project):
        """Import configuration files from user project."""

        nemoa.log('import private resources')
        nemoa.setLog(indent='+1')

        # check if project exists
        if not project in self.__listUserWorkspaces():
            nemoa.log(
                'warning', """
                could not open project '%s':
                project folder could not be found in '%s'!
                """ % (project, self.__basepath['user']))
            return False

        # set project
        self.__project = project

        # update paths
        self.__updatePaths(base='user')

        # update path of cache
        self.__updateCachePaths()

        # update logger to current logfile
        nemoa.initLogger(logfile=self.__path['logfile'])

        # import object configurations for current project
        self.__scanForConfigFiles()
        self.__scanForScripts()
        self.__scanForNetworks()

        nemoa.setLog(indent='-1')
        return True
Exemplo n.º 3
0
    def __importConfigFile(self, file):
        """Import configuration (.ini) file."""

        # search definition file
        if os.path.isfile(file):
            configFile = file
        elif os.path.isfile(self.__basepath['workspace'] + file):
            configFile = self.__basepath['workspace'] + file
        else:
            nemoa.log('warning', "configuration file '%s' does not exist!" % (file))
            return False

        # logger info
        nemoa.log("parsing configuration file: '" + configFile + "'")
        nemoa.setLog(indent = '+1')

        # import and register objects without testing
        importer = configFileImporter(self)
        objConfList = importer.load(configFile)

        for objConf in objConfList:
            self.__addObjToStore(objConf)

        self.__check(objConfList)

        nemoa.setLog(indent = '-1')
        return True
Exemplo n.º 4
0
 def configure(self):
     """Configure model."""
     nemoa.log('configure model \'%s\'' % (self.name()))
     nemoa.setLog(indent = '+1')
     if not 'check' in self.__config:
         self.__config['check'] = {'dataset': False,
             'network': False, 'System': False}
     self.__config['check']['dataset'] = \
         self.dataset.configure(network = self.network)
     if not self.__config['check']['dataset']:
         nemoa.log('error', """could not configure model: dataset
             could not be configured!""")
         nemoa.setLog(indent = '-1')
         return False
     self.__config['check']['network'] = \
         self.network.configure(dataset = self.dataset,
             system = self.system)
     if not self.__config['check']['network']:
         nemoa.log('error', """could not configure model: network
             could not be configured!""")
         nemoa.setLog(indent = '-1')
         return False
     self.__config['check']['system'] = \
         self.system.configure(network = self.network,
             dataset = self.dataset)
     if not self.__config['check']['system']:
         nemoa.log('error', """could not configure model: system
             could not be configured!""")
         nemoa.setLog(indent = '-1')
         return False
     nemoa.setLog(indent = '-1')
     return True
Exemplo n.º 5
0
    def loadProject(self, project):
        """Import configuration files from user project."""

        nemoa.log('import private resources')
        nemoa.setLog(indent = '+1')

        # check if project exists
        if not project in self.__listUserWorkspaces():
            nemoa.log('warning', """
                could not open project '%s':
                project folder could not be found in '%s'!
                """ % (project, self.__basepath['user']))
            return False
        
        # set project
        self.__project = project

        # update paths
        self.__updatePaths(base = 'user')

        # update path of cache
        self.__updateCachePaths()

        # update logger to current logfile
        nemoa.initLogger(logfile = self.__path['logfile'])

        # import object configurations for current project
        self.__scanForConfigFiles()
        self.__scanForScripts()
        self.__scanForNetworks()

        nemoa.setLog(indent = '-1')
        return True
Exemplo n.º 6
0
    def __importConfigFile(self, file):
        """Import configuration (.ini) file."""

        # search definition file
        if os.path.isfile(file):
            configFile = file
        elif os.path.isfile(self.__basepath['workspace'] + file):
            configFile = self.__basepath['workspace'] + file
        else:
            nemoa.log('warning',
                      "configuration file '%s' does not exist!" % (file))
            return False

        # logger info
        nemoa.log("parsing configuration file: '" + configFile + "'")
        nemoa.setLog(indent='+1')

        # import and register objects without testing
        importer = configFileImporter(self)
        objConfList = importer.load(configFile)

        for objConf in objConfList:
            self.__addObjToStore(objConf)

        self.__check(objConfList)

        nemoa.setLog(indent='-1')
        return True
Exemplo n.º 7
0
    def __importModelFromFile(self, file):
        """Return new model instance and set configuration and parameters from file."""

        nemoa.log('import model from file')
        nemoa.setLog(indent = '+1')

        # check file
        if not os.path.exists(file):
            if os.path.exists(
                nemoa.workspace.path('models') + file + '.mp'):
                file = nemoa.workspace.path('models') + file + '.mp'
            else: return nemoa.log('error', """
                could not load model '%s':
                file does not exist.""" % file)

        # load model parameters and configuration from file
        nemoa.log('load model: \'%s\'' % file)
        modelDict = nemoa.common.dictFromFile(file)

        model = self.__getModelInstance(
            name    = modelDict['config']['name'],
            config  = modelDict['config'],
            dataset = modelDict['dataset']['cfg'],
            network = modelDict['network']['cfg'],
            system  = modelDict['system']['config'])

        if nemoa.type.isModel(model): model._set(modelDict)
        else: return None 

        nemoa.setLog(indent = '-1')
        return model
Exemplo n.º 8
0
def scripts(project):
    """Print list of scripts to standard output."""
    logParams = nemoa.getLog()
    nemoa.setLog(mode='silent')
    workspace = nemoa.open(project)
    nemoa.setLog(mode=logParams['mode'])
    scripts = workspace.list(type='script', namespace=workspace.project())
    print 'Scripts in project %s:\n' % (project)
    for script in scripts:
        print '    %s' % (script)
    print ''
Exemplo n.º 9
0
def scripts(project):
    """Print list of scripts to standard output."""
    logParams = nemoa.getLog()
    nemoa.setLog(mode="silent")
    workspace = nemoa.open(project)
    nemoa.setLog(mode=logParams["mode"])
    scripts = workspace.list(type="script", namespace=workspace.project())
    print "Scripts in project %s:\n" % (project)
    for script in scripts:
        print "    %s" % (script)
    print ""
Exemplo n.º 10
0
    def __getModelInstance(self,
                           name=None,
                           config=None,
                           dataset=None,
                           network=None,
                           system=None):
        """Return new model instance."""

        nemoa.log('create model instance')
        nemoa.setLog(indent='+1')

        # create dataset instance (if not given)
        if not nemoa.type.isDataset(dataset):            dataset = \
self.__getInstance(type = 'dataset', config = dataset)
        if not nemoa.type.isDataset(dataset):
            nemoa.log('error',
                      'could not create model instance: dataset is invalid!')
            nemoa.setLog(indent='-1')
            return None

        # create network instance (if not given)
        if network == None: network = {'type': 'auto'}
        if not nemoa.type.isNetwork(network):            network = \
self.__getInstance(type = 'network', config = network)
        if not nemoa.type.isNetwork(network):
            nemoa.log('error',
                      'could not create model instance: network is invalid!')
            nemoa.setLog(indent='-1')
            return None

        # create system instance (if not given)
        if not nemoa.type.isSystem(system):            system = \
self.__getInstance(type = 'system', config = system)
        if not nemoa.type.isSystem(system):
            nemoa.log('error',
                      'could not create model instance: system is invalid!')
            nemoa.setLog(indent='-1')
            return None

        # create name string (if not given)
        if name == None:
            name = '-'.join([dataset.name(), network.name(), system.name()])

        # create model instance
        model = self.__getInstance(type='model',
                                   config=config,
                                   name=name,
                                   dataset=dataset,
                                   network=network,
                                   system=system)

        nemoa.setLog(indent='-1')
        return model
Exemplo n.º 11
0
    def __getInstance(self, type = None, config = None, empty = False, **kwargs):
        """Return new instance of given object type and configuration."""
        nemoa.log('create%s %s instance' % (' empty' if empty else '', type))
        nemoa.setLog(indent = '+1')

        # import module
        module = importlib.import_module('nemoa.' + str(type))

        # get objects configuration as dictionary
        config = nemoa.workspace.getConfig(type = type,
            config = config, **kwargs)
        if not isinstance(config, dict):
            nemoa.log('error', """could not create %s instance:
                unknown configuration!""" % (type))
            nemoa.setLog(indent = '-1')
            return None

        # create and initialize new instance of given class
        instance = module.empty() if empty \
            else module.new(config = config, **kwargs)

        # check instance class
        if not nemoa.type.isInstanceType(instance, type):
            nemoa.log('error', """could not create %s instance:
                invalid configuration!""" % (type))
            nemoa.setLog(indent = '-1')
            return None

        nemoa.log('name of %s is: \'%s\'' % (type, instance.name()))
        nemoa.setLog(indent = '-1')
        return instance
Exemplo n.º 12
0
 def configure(self, dataset=None, network=None, *args, **kwargs):
     """Configure system and subsystems to network and dataset."""
     if not hasattr(self.__class__, "_configure") or not callable(getattr(self.__class__, "_configure")):
         return True
     nemoa.log("configure system '%s'" % (self.name()))
     nemoa.setLog(indent="+1")
     if not self.checkNetwork(network):
         nemoa.log(
             "error",
             """
             system could not be configured:
             network is not valid!""",
         )
         nemoa.setLog(indent="-1")
         return False
     if not self.checkDataset(dataset):
         nemoa.log(
             "error",
             """
             system could not be configured:
             dataset is not valid!""",
         )
         nemoa.setLog(indent="-1")
         return False
     retVal = self._configure(dataset=dataset, network=network, *args, **kwargs)
     nemoa.setLog(indent="-1")
     return retVal
Exemplo n.º 13
0
    def __getInstance(self, type=None, config=None, empty=False, **kwargs):
        """Return new instance of given object type and configuration."""
        nemoa.log('create%s %s instance' % (' empty' if empty else '', type))
        nemoa.setLog(indent='+1')

        # import module
        module = importlib.import_module('nemoa.' + str(type))

        # get objects configuration as dictionary
        config = nemoa.workspace.getConfig(type=type, config=config, **kwargs)
        if not isinstance(config, dict):
            nemoa.log(
                'error', """could not create %s instance:
                unknown configuration!""" % (type))
            nemoa.setLog(indent='-1')
            return None

        # create and initialize new instance of given class
        instance = module.empty() if empty \
            else module.new(config = config, **kwargs)

        # check instance class
        if not nemoa.type.isInstanceType(instance, type):
            nemoa.log(
                'error', """could not create %s instance:
                invalid configuration!""" % (type))
            nemoa.setLog(indent='-1')
            return None

        nemoa.log('name of %s is: \'%s\'' % (type, instance.name()))
        nemoa.setLog(indent='-1')
        return instance
Exemplo n.º 14
0
    def __scanForNetworks(self, files=None):
        """Scan for scripts files in current project."""

        nemoa.log('scanning for networks')
        nemoa.setLog(indent='+1')

        # are files given?
        if files == None:
            files = self.__path['networks'] + '*.ini'

        # import definition files
        for file in glob.iglob(self.getPath(files)):
            self.__registerNetwork(file)

        nemoa.setLog(indent='-1')

        return True
Exemplo n.º 15
0
    def __scanForNetworks(self, files = None):
        """Scan for scripts files in current project."""

        nemoa.log('scanning for networks')
        nemoa.setLog(indent = '+1')

        # are files given?
        if files == None:
            files = self.__path['networks'] + '*.ini'

        # import definition files
        for file in glob.iglob(self.getPath(files)):
            self.__registerNetwork(file)

        nemoa.setLog(indent = '-1')

        return True
Exemplo n.º 16
0
    def __scanForConfigFiles(self, files = None):
        """Import all config files from current project."""

        nemoa.log('scanning for configuration files')
        nemoa.setLog(indent = '+1')

        # are files given?
        if files == None:
            files = self.__path['workspace'] + '*.ini'

        # import configuration files
        for file in glob.iglob(self.getPath(files)):
            self.__importConfigFile(file)

        nemoa.setLog(indent = '-1')

        return True
Exemplo n.º 17
0
    def __getModelInstance(self, name = None, config = None,
        dataset = None, network = None, system = None):
        """Return new model instance."""

        nemoa.log('create model instance')
        nemoa.setLog(indent = '+1')

        # create dataset instance (if not given)
        if not nemoa.type.isDataset(dataset): dataset = \
            self.__getInstance(type = 'dataset', config = dataset)
        if not nemoa.type.isDataset(dataset): 
            nemoa.log('error',
                'could not create model instance: dataset is invalid!')
            nemoa.setLog(indent = '-1')
            return None

        # create network instance (if not given)
        if network == None: network = {'type': 'auto'}
        if not nemoa.type.isNetwork(network): network = \
            self.__getInstance(type = 'network', config = network)
        if not nemoa.type.isNetwork(network): 
            nemoa.log('error',
                'could not create model instance: network is invalid!')
            nemoa.setLog(indent = '-1')
            return None

        # create system instance (if not given)
        if not nemoa.type.isSystem(system): system = \
            self.__getInstance(type = 'system', config = system)
        if not nemoa.type.isSystem(system):
            nemoa.log('error',
                'could not create model instance: system is invalid!')
            nemoa.setLog(indent = '-1')
            return None

        # create name string (if not given)
        if name == None: name = '-'.join(
            [dataset.name(), network.name(), system.name()])

        # create model instance
        model = self.__getInstance(
            type = 'model', config = config, name = name,
            dataset = dataset, network = network, system = system)

        nemoa.setLog(indent = '-1')
        return model
Exemplo n.º 18
0
    def __scanForConfigFiles(self, files=None):
        """Import all config files from current project."""

        nemoa.log('scanning for configuration files')
        nemoa.setLog(indent='+1')

        # are files given?
        if files == None:
            files = self.__path['workspace'] + '*.ini'

        # import configuration files
        for file in glob.iglob(self.getPath(files)):
            self.__importConfigFile(file)

        nemoa.setLog(indent='-1')

        return True
Exemplo n.º 19
0
    def optimizeParams(self, dataset, schedule):
        """Optimize system parameters using data and given schedule."""

        # check if optimization schedule exists for current system
        # and merge default, existing and given schedule
        if not "params" in schedule:
            config = self._default("optimize")
            nemoa.common.dictMerge(self._config["optimize"], config)
            self._config["optimize"] = config
        elif not self.getType() in schedule["params"]:
            return nemoa.log(
                "error",
                """could not optimize model:
            optimization schedule '%s' does not include system '%s'
            """
                % (schedule["name"], self.getType()),
            )
        else:
            config = self._default("optimize")
            nemoa.common.dictMerge(self._config["optimize"], config)
            nemoa.common.dictMerge(schedule["params"][self.getType()], config)
            self._config["optimize"] = config

        ################################################################
        # System independent optimization settings                     #
        ################################################################

        # check dataset
        if (not "checkDataset" in config or config["checkDataset"] == True) and not self._checkDataset(dataset):
            return False

        # initialize inspector
        inspector = nemoa.system.base.inspector(self)
        if "inspect" in config and not config["inspect"] == False:
            inspector.setTestData(self._getTestData(dataset))

        # optimize system parameters
        algorithm = config["algorithm"].title()
        nemoa.log("note", "optimize '%s' (%s) using algorithm '%s'" % (self.name(), self.getType(), algorithm))
        nemoa.setLog(indent="+1")
        retVal = self._optimizeParams(dataset, schedule, inspector)
        nemoa.setLog(indent="-1")

        return retVal
Exemplo n.º 20
0
    def plot(self, plot = None, output = 'file', file = None, **kwargs):
        """Create plot of model."""

        nemoa.log('create plot of model')
        nemoa.setLog(indent = '+1')

        # check if model is configured
        if not self.__isConfigured():
            nemoa.log('error', """could not create plot of model:
                model is not yet configured!""")
            nemoa.setLog(indent = '-1')
            return False

        # get plot instance
        nemoa.log('create plot instance')
        nemoa.setLog(indent = '+1')

        if plot == None: plot = self.system.getType() + '.default'
        if isinstance(plot, str):
            plotName, plotParams = nemoa.common.strSplitParams(plot)
            mergeDict = plotParams
            for param in kwargs.keys(): plotParams[param] = kwargs[param]
            objPlot = self.__getPlot(name = plotName, params = plotParams)
            if not objPlot:
                nemoa.log('warning', "could not create plot: unknown configuration '%s'" % (plotName))
                nemoa.setLog(indent = '-1')
                return None
        elif isinstance(plot, dict): objPlot = self.__getPlot(config = plot)
        else: objPlot = self.__getPlot()
        if not objPlot: return None

        # prepare filename
        if output == 'display': file = None
        elif output == 'file' and not file:
            file = nemoa.common.getEmptyFile(nemoa.workspace.path('plots') + \
                self.name() + '/' + objPlot.cfg['name'] + \
                '.' + objPlot.settings['fileformat'])

        # create plot
        retVal = objPlot.create(self, file = file)
        if not file == None: nemoa.log('save plot: ' + file)
        
        nemoa.setLog(indent = '-2')
        return retVal
Exemplo n.º 21
0
    def preprocessData(self, **kwargs):
        """Data preprocessing.

        Keyword Arguments:
            stratify -- see algorithm in stratifyData
            normalize -- see algorithm in normalizeData
            transform -- see algorithm in transformData

        Description:
            Process stratification, normalization and transformation."""

        nemoa.log('preprocessing data')
        nemoa.setLog(indent = '+1')
        if 'stratify'  in kwargs.keys(): self.stratifyData(kwargs['stratify'])
        if 'normalize' in kwargs.keys(): self.normalizeData(kwargs['normalize'])
        if 'transform' in kwargs.keys(): self.transformData(kwargs['transform'])
        nemoa.setLog(indent = '-1')

        return True
Exemplo n.º 22
0
    def configure(self, dataset, system):
        """Configure network to dataset and system."""

        # check if network instance is empty
        if self.isEmpty():
            nemoa.log('configuration is not needed: network is \'empty\'')
            return True

        # check if dataset instance is available
        if not nemoa.type.isDataset(dataset):
            nemoa.log('error', 'could not configure network: no valid dataset instance given!')
            return False
 
         # check if system instance is available
        if not nemoa.type.isSystem(system):
            nemoa.log('error', 'could not configure network: no valid system instance given!')
            return False

        nemoa.log('configure network: \'%s\'' % (self.name()))
        nemoa.setLog(indent = '+1')

        # type: 'auto is used for networks
        # wich are created by datasets (visible units)
        # and systems (hidden units)
        if self.cfg['type'] == 'auto':
            self.__getVisibleNodesFromDataset(dataset)
            self.__getHiddenNodesFromSystem(system)
            self.__getEdgesFromNodesAndLayers()
            self.updateGraph()
            nemoa.setLog(indent = '-1')
            return True

        # type: 'autolayer' is used for networks
        # wich are created by layers and sizes
        if self.cfg['type'] == 'autolayer':
            self.__getNodesFromLayers()
            self.__getEdgesFromNodesAndLayers()
            self.updateGraph()
            nemoa.setLog(indent = '-1')
            return True

        # configure network to dataset
        groups = dataset.getColGroups()
        changes = []
        for group in groups:
            if not group in self.cfg['nodes'] \
                or not (groups[group] == self.cfg['nodes'][group]):
                self.updateGraph(nodelist = {'type': group, 'list': groups[group]})

        nemoa.setLog(indent = '-1')
        return True
Exemplo n.º 23
0
    def optimizeParams(self, dataset, schedule):
        """Optimize system parameters using data and given schedule."""

        # check if optimization schedule exists for current system
        # and merge default, existing and given schedule 
        if not 'params' in schedule:
            config = self._default('optimize')
            nemoa.common.dictMerge(self._config['optimize'], config)
            self._config['optimize'] = config    
        elif not self.getType() in schedule['params']: return nemoa.log(
            'error', """could not optimize model:
            optimization schedule '%s' does not include system '%s'
            """ % (schedule['name'], self.getType()))
        else:
            config = self._default('optimize')
            nemoa.common.dictMerge(self._config['optimize'], config)
            nemoa.common.dictMerge(schedule['params'][self.getType()], config)
            self._config['optimize'] = config

        ################################################################
        # System independent optimization settings                     #
        ################################################################

        # check dataset
        if (not 'checkDataset' in config or config['checkDataset'] == True) \
            and not self._checkDataset(dataset): return False

        # initialize inspector
        inspector = nemoa.system.base.inspector(self)
        if 'inspect' in config and not config['inspect'] == False:
            inspector.setTestData(self._getTestData(dataset)) 

        # optimize system parameters
        algorithm = config['algorithm'].title()
        nemoa.log('note', "optimize '%s' (%s) using algorithm '%s'" % \
            (self.name(), self.getType(), algorithm))
        nemoa.setLog(indent = '+1')
        retVal = self._optimizeParams(dataset, schedule, inspector)
        nemoa.setLog(indent = '-1')

        return retVal
Exemplo n.º 24
0
    def save(self, file = None):
        """Save model settings to file and return filepath."""

        nemoa.log('save model to file')
        nemoa.setLog(indent = '+1')

        # get filename
        if file == None:
            fileName = '%s.mp' % (self.name())
            filePath = nemoa.workspace.path('models')
            file = filePath + fileName
        file = nemoa.common.getEmptyFile(file)

        # save model parameters and configuration to file
        nemoa.common.dictToFile(self._get(), file)

        # create console message
        nemoa.log("save model as: '%s'" % (os.path.basename(file)[:-3]))

        nemoa.setLog(indent = '-1')
        return file
Exemplo n.º 25
0
    def export(self, file, **kwargs):
        """Export data to file."""

        file = nemoa.common.getEmptyFile(file)
        type = nemoa.common.getFileExt(file).lower()

        nemoa.log('export data to file')
        nemoa.setLog(indent = '+1')

        nemoa.log('exporting data to file: \'%s\'' % (file))
        if type in ['gz', 'data']: retVal = self.save(file)
        elif type in ['csv', 'tsv', 'tab', 'txt']:
            cols, data = self.getData(output = ('cols', 'recarray'))
            retVal = nemoa.common.csvSaveData(file, data,
                cols = [''] + cols, **kwargs)
        else: retVal = nemoa.log('error', """
            could not export dataset:
            unsupported file type '%s'""" % (type))

        nemoa.setLog(indent = '-1')
        return retVal
Exemplo n.º 26
0
    def __createNewModel(self,
                         name,
                         config=None,
                         dataset=None,
                         network=None,
                         system=None,
                         configure=True,
                         initialize=True):
        nemoa.log('create new model')
        nemoa.setLog(indent='+1')

        model = self.__getModelInstance(name=name,
                                        config=config,
                                        dataset=dataset,
                                        network=network,
                                        system=system)

        if not nemoa.type.isModel(model):
            nemoa.log('error', 'could not create new model!')
            nemoa.setLog(indent='-1')
            return False

        # configure model
        if configure: model.configure()

        # initialize model parameters
        if initialize: model.initialize()

        nemoa.setLog(indent='-1')
        return model
Exemplo n.º 27
0
    def loadCommon(self):
        """Import common projects."""

        nemoa.log('import shared resources')
        nemoa.setLog(indent = '+1')

        # get current project
        curProject = self.__project

        # 
        for project in self.__listCommonProjects():

            # set common project, update paths and import workspaces
            self.__project = project
            self.__updatePaths(base = 'common')
            self.__scanForConfigFiles()
            self.__scanForScripts()
            self.__scanForNetworks()

        # reset to previous project
        self.__project = curProject

        nemoa.setLog(indent = '-1')
        return True
Exemplo n.º 28
0
    def loadCommon(self):
        """Import common projects."""

        nemoa.log('import shared resources')
        nemoa.setLog(indent='+1')

        # get current project
        curProject = self.__project

        #
        for project in self.__listCommonProjects():

            # set common project, update paths and import workspaces
            self.__project = project
            self.__updatePaths(base='common')
            self.__scanForConfigFiles()
            self.__scanForScripts()
            self.__scanForNetworks()

        # reset to previous project
        self.__project = curProject

        nemoa.setLog(indent='-1')
        return True
Exemplo n.º 29
0
 def configure(self, dataset = None, network = None, *args, **kwargs):
     """Configure system and subsystems to network and dataset."""
     if not hasattr(self.__class__, '_configure') \
         or not callable(getattr(self.__class__, '_configure')):
         return True
     nemoa.log("configure system '%s'" % (self.name()))
     nemoa.setLog(indent = '+1')
     if not self.checkNetwork(network):
         nemoa.log('error', """
             system could not be configured:
             network is not valid!""")
         nemoa.setLog(indent = '-1')
         return False
     if not self.checkDataset(dataset):
         nemoa.log('error', """
             system could not be configured:
             dataset is not valid!""")
         nemoa.setLog(indent = '-1')
         return False
     retVal = self._configure(dataset = dataset, network = network,
         *args, **kwargs)
     nemoa.setLog(indent = '-1')
     return retVal
Exemplo n.º 30
0
    def __createNewModel(self, name, config = None,
        dataset = None, network = None, system = None,
        configure = True, initialize = True):
        nemoa.log('create new model')
        nemoa.setLog(indent = '+1')

        model = self.__getModelInstance(name = name, config  = config,
            dataset = dataset, network = network, system  = system)

        if not nemoa.type.isModel(model):
            nemoa.log('error', 'could not create new model!')
            nemoa.setLog(indent = '-1')
            return False

        # configure model
        if configure: model.configure()

        # initialize model parameters
        if initialize: model.initialize()

        nemoa.setLog(indent = '-1')
        return model
Exemplo n.º 31
0
def main(argv):
    os.system('cls' if os.name == 'nt' else 'clear')
    nemoa.setLog(mode='shell')
    nemoa.welcome()
Exemplo n.º 32
0
def main(argv):
    os.system('cls' if os.name=='nt' else 'clear')
    nemoa.setLog(mode = 'shell')
    nemoa.welcome()
Exemplo n.º 33
0
    def optimize(self, schedule = None, **kwargs):
        """Optimize system parameters."""

        nemoa.log('optimize model')
        nemoa.setLog(indent = '+1')

        # check if model is empty
        if self.isEmpty():
            nemoa.log('warning', "empty models can not be optimized!")
            nemoa.setLog(indent = '-1')
            return self

        # check if model is configured
        if not self.__isConfigured():
            nemoa.log('error',
                'could not optimize model: model is not yet configured!')
            nemoa.setLog(indent = '-1')
            return False

        # get optimization schedule
        if schedule == None: schedule = self.system.getType() + '.default'
        elif not '.' in schedule: schedule = \
            self.system.getType() + '.' + schedule
        schedule = nemoa.workspace.getConfig(
            type = 'schedule', config = schedule,
            merge = ['params', self.system.getType()],
            **kwargs)
        if not schedule:
            nemoa.log('error', """
                could not optimize system parameters:
                optimization schedule is not valid!""")
            nemoa.setLog(indent = '-1')
            return self
        
        # optimization of system parameters
        nemoa.log("starting optimization schedule: '%s'" % (schedule['name']))
        nemoa.setLog(indent = '+1')

        # 2DO: find better solution for multistage optimization
        if 'stage' in schedule and len(schedule['stage']) > 0:
            for stage, params in enumerate(config['stage']):
                self.system.optimizeParams(self.dataset, **params)
        elif 'params' in schedule:
            self.system.optimizeParams(
                dataset = self.dataset, schedule = schedule)

        nemoa.setLog(indent = '-1')

        # update network
        self.system.updateNetwork(self.network)

        nemoa.setLog(indent = '-1')
        return self
Exemplo n.º 34
0
    def transformData(self, algorithm = 'system', system = None, mapping = None, **kwargs):
        """Transform dataset.

        Keyword Arguments:
            algorithm -- name of algorithm used for data transformation
                'system':
                    Transform data using nemoa system instance
                'gaussToBinary':
                    Transform Gauss distributed values to binary values in {0, 1}
                'gaussToWeight':
                    Transform Gauss distributed values to weights in [0, 1]
                'gaussToDistance': ??
                    Transform Gauss distributed values to distances in [0, 1]
            system -- nemoa system instance (nemoa object root class 'system')
                used for model based transformation of data
            mapping -- ..."""

        if not isinstance(algorithm, str): return False

        # system based data transformation
        if algorithm.lower() == 'system':
            if not nemoa.type.isSystem(system): return nemoa.log('error',
                """could not transform data using system:
                parameter 'system' is invalid!""")
            nemoa.log('transform data using system \'%s\'' % (system.name()))
            nemoa.setLog(indent = '+1')
            
            if mapping == None: mapping = system.getMapping()

            sourceColumns = system.getUnits(group = mapping[0])[0]
            targetColumns = system.getUnits(group = mapping[-1])[0]

            self.setColLabels(sourceColumns)
            for src in self.data:
                data = self.data[src]['array']
                dataArray = data[sourceColumns].view('<f8').reshape(
                    data.size, len(sourceColumns))
                transArray = system.mapData(dataArray, mapping = mapping, **kwargs)

                # create empty record array
                numRows = self.data[src]['array']['label'].size
                colNames = ('label',) + tuple(targetColumns)
                colFormats = ('<U12',) + tuple(['<f8' for x in targetColumns])
                newRecArray = numpy.recarray((numRows,),
                    dtype = zip(colNames, colFormats))

                # set values in record array
                newRecArray['label'] = self.data[src]['array']['label']
                for colID, colName in enumerate(newRecArray.dtype.names[1:]):

                    # update source data columns
                    newRecArray[colName] = (transArray[:, colID]).astype(float)

                # set record array
                self.data[src]['array'] = newRecArray

            self.setColLabels(targetColumns)
            nemoa.setLog(indent = '-1')
            return True

        # gauss to binary data transformation
        elif algorithm.lower() in ['gausstobinary', 'binary']:
            nemoa.log('transform data using \'%s\'' % (algorithm))
            for src in self.data:
                # update source per column (recarray)
                for colName in self.data[src]['array'].dtype.names[1:]:
                    # update source data columns
                    self.data[src]['array'][colName] = \
                        (self.data[src]['array'][colName] > 0.0).astype(float)
            return True

        # gauss to weight in [0, 1] data transformation
        elif algorithm.lower() in ['gausstoweight', 'weight']:
            nemoa.log('transform data using \'%s\'' % (algorithm))
            for src in self.data:
                # update source per column (recarray)
                for colName in self.data[src]['array'].dtype.names[1:]:
                    # update source data columns
                    self.data[src]['array'][colName] = \
                        (2.0 / (1.0 + numpy.exp(-1.0 * \
                        self.data[src]['array'][colName] ** 2))).astype(float)
            return True

        # gauss to distance data transformation
        # ????
        elif algorithm.lower() in ['gausstodistance', 'distance']:
            nemoa.log('transform data using \'%s\'' % (algorithm))
            for src in self.data:
                # update source per column (recarray)
                for colName in self.data[src]['array'].dtype.names[1:]:
                    # update source data columns
                    self.data[src]['array'][colName] = \
                        (1.0 - (2.0 / (1.0 + numpy.exp(-1.0 * \
                        self.data[src]['array'][colName] ** 2)))).astype(float)
            return True

        return nemoa.log('error'
            "could not transform data: unknown algorithm '%s'!" % (algorithm))
Exemplo n.º 35
0
    def configure(self, network, useCache = False, **kwargs):
        """Configure dataset to a given network object

        Keyword arguments:
            network -- nemoa network object
            useCache -- shall data be cached"""

        nemoa.log("configure dataset '%s' to network '%s'" % \
            (self.name(), network.name()))
        nemoa.setLog(indent = '+1')

        # load data from cachefile (if caching is used and cachefile exists)
        cacheFile = self.searchCacheFile(network) if useCache else None
        if cacheFile and self.load(cacheFile):
            nemoa.log('load cachefile: \'%s\'' % (cacheFile))

            # preprocess data
            if 'preprocessing' in self.cfg.keys():
                self.preprocessData(**self.cfg['preprocessing'])
            nemoa.setLog(indent = '-1')
            return True

        # create table with one record for every single dataset files 
        if not 'table' in self.cfg:
            conf = self.cfg.copy()
            self.cfg['table'] = {}
            self.cfg['table'][self.cfg['name']] = conf
            self.cfg['table'][self.cfg['name']]['fraction'] = 1.0

        ################################################################
        # Annotation                                                   #
        ################################################################

        # get nodes from network and convert to common format
        if network.cfg['type'] == 'auto': netGroups = {'v': None}
        else:
            # get grouped network node labels and label format
            netGroups = network.getNodeGroups(type = 'visible')

            netGroupsOrder = []
            for layer in netGroups:
                netGroupsOrder.append((network.layer(layer)['id'], layer))
            netGroupsOrder = sorted(netGroupsOrder)

            # convert network node labels to common format
            nemoa.log('search network nodes in dataset sources')
            convNetGroups = {}
            convNetGroupsLost = {}
            convNetNodes = []
            convNetNodesLost = []
            netLblFmt = network.cfg['label_format']
            for id, group in netGroupsOrder:
                convNetGroups[group], convNetGroupsLost[group] = \
                    nemoa.annotation.convert(netGroups[group], input = netLblFmt)
                convNetNodes += convNetGroups[group]
                convNetNodesLost += convNetGroupsLost[group]

            # notify if any network node labels could not be converted
            if convNetNodesLost:
                nemoa.log('%s of %s network nodes could not be converted! (see logfile)'
                    % (len(convNetNodesLost), len(convNetNodes)))
                ## 2DO get original node labels for log file
                nemoa.log('logfile', nemoa.common.strToList(convNetNodesLost))

        # get columns from dataset files and convert to common format
        colLabels = {}
        nemoa.log('configure data sources')
        nemoa.setLog(indent = '+1')
        for src in self.cfg['table']:
            nemoa.log("configure '%s'" % (src))
            srcCnf = self.cfg['table'][src]

            # get column labels from csv-file
            if 'csvtype' in srcCnf['source']:
                csvType = srcCnf['source']['csvtype'].strip().lower()
            else: csvType = None
            origColLabels = nemoa.common.csvGetColLabels(
                srcCnf['source']['file'], type = csvType)
            if not origColLabels: continue

            # set annotation format
            format = srcCnf['source']['columns'] \
                if 'columns' in srcCnf['source'] else 'generic:string'

            # convert column labes
            convColLabels, convColLabelsLost = \
                nemoa.annotation.convert(origColLabels, input = format)

            # notify if any dataset columns could not be converted
            if convColLabelsLost:
                nemoa.log('warning', 
                    "%i of %i dataset columns could not be converted! (logfile)"
                        % (len(convColLabelsLost), len(convColLabels)))
                nemoa.log("logfile", ", ".join([convColLabels[i] \
                    for i in convColLabelsLost]))

            if not network.cfg['type'] == 'auto':

                # search converted network nodes in converted column labels
                numLost = 0
                numAll = 0
                lostNodes = {}
                for id, group in netGroupsOrder:
                    lostNodesConv = \
                        [val for val in convNetGroups[group] \
                        if val not in convColLabels]
                    numAll += len(convNetGroups[group])
                    if not lostNodesConv: continue
                    numLost += len(lostNodesConv)

                    # get original labels
                    lostNodes[group] = [netGroups[group][
                        convNetGroups[group].index(val)] for val in lostNodesConv]

                # notify if any network nodes could not be found
                if numLost:
                    nemoa.log('warning', """
                        %i of %i network nodes could not be found in
                        dataset source! (logfile)""" % (numLost, numAll))
                    for group in lostNodes:
                        nemoa.log("logfile", """
                            missing nodes (group %s): """ % (group)
                            + ", ".join(lostNodes[group]))

            # prepare dictionary for column source ids
            colLabels[src] = {
                'conv': convColLabels,
                'usecols': (),
                'notusecols': convColLabelsLost }

        nemoa.setLog(indent = '-1')

        # intersect converted dataset column labels
        interColLabels = colLabels[colLabels.keys()[0]]['conv']
        for src in colLabels:
            list = colLabels[src]['conv']
            blackList = [list[i] for i in colLabels[src]['notusecols']]
            interColLabels = [val for val in interColLabels \
                if val in list and not val in blackList]

        # if network type is 'auto', set network visible nodes
        # to intersected data from database files (without label column)
        if network.cfg['type'] == 'auto':
            netGroups['v'] = [label for label in interColLabels \
                if not label == 'label']
            convNetGroups = netGroups

        # search network nodes in dataset columns
        self.cfg['columns'] = ()
        for groupid, group in netGroupsOrder:
            found = 0

            for id, col in enumerate(convNetGroups[group]):
                if not col in interColLabels:
                    continue
                found += 1

                # add column (use network label and group)
                self.cfg['columns'] += ((group, netGroups[group][id]), )
                for src in colLabels: colLabels[src]['usecols'] \
                    += (colLabels[src]['conv'].index(col), )
            if not found:
                nemoa.log('error', """
                    no node from network group '%s'
                    could be found in dataset source!
                    """ % (group))
                nemoa.setLog(indent = '-1')
                return False

        # update source file config
        for src in colLabels: self.cfg['table'][src]['source']['usecols'] \
            = colLabels[src]['usecols']

        ################################################################
        # Column & Row Filters                                         #
        ################################################################

        # add column filters and partitions from network node groups
        self.cfg['colFilter'] = {'*': ['*:*']}
        self.cfg['colPartitions'] = {'groups': []}
        for group in netGroups:
            self.cfg['colFilter'][group] = [group + ':*']
            self.cfg['colPartitions']['groups'].append(group)

        # add row filters and partitions from sources
        self.cfg['rowFilter'] = {'*': ['*:*']}
        self.cfg['rowPartitions'] = {'source': []}
        for source in self.cfg['table']:
            self.cfg['rowFilter'][source] = [source + ':*']
            self.cfg['rowPartitions']['source'].append(source)

        ################################################################
        # Import data from CSV-files into numpy arrays                 #
        ################################################################

        # import data from sources
        nemoa.log('import data from sources')
        nemoa.setLog(indent = '+1')
        self.data = {}
        for src in self.cfg['table']:
            self.data[src] = {
                'fraction': self.cfg['table'][src]['fraction'],
                'array': self.csvGetData(src) }
        nemoa.setLog(indent = '-1')

        # save cachefile
        if useCache:
            cacheFile = self.createCacheFile(network)
            nemoa.log('save cachefile: \'%s\'' % (cacheFile))
            self.save(cacheFile)

        # preprocess data
        if 'preprocessing' in self.cfg.keys():
            self.preprocessData(**self.cfg['preprocessing'])

        nemoa.setLog(indent = '-1')
        return True