示例#1
0
    def __importConfigFile(self, file):
        """Import configuration (.ini) file."""

        # search definition file
        if os.path.isfile(file):
            configFile = file
        elif os.path.isfile(self.__basepath['workspace'] + file):
            configFile = self.__basepath['workspace'] + file
        else:
            nemoa.log('warning',
                      "configuration file '%s' does not exist!" % (file))
            return False

        # logger info
        nemoa.log("parsing configuration file: '" + configFile + "'")
        nemoa.setLog(indent='+1')

        # import and register objects without testing
        importer = configFileImporter(self)
        objConfList = importer.load(configFile)

        for objConf in objConfList:
            self.__addObjToStore(objConf)

        self.__check(objConfList)

        nemoa.setLog(indent='-1')
        return True
示例#2
0
文件: base.py 项目: Goatbuster/nemoa
    def _is_compatible_lff(self):
        """Test compatibility to layered feed forward networks.

        Returns:
            Boolean value which is True if the following conditions are
            satisfied:
            (1) All layers of the network are not empty
            (2) The first and last layer of the network are visible,
                the layers between them are hidden

        """

        # test if network contains empty layers
        for layer in self._get_layers():
            if not len(self._get_layer(layer)['nodes']) > 0:
                return nemoa.log(
                    'error', """Feedforward networks do
                    not allow empty layers.""")

        # test if and only if the first and the last layer are visible
        for layer in self._get_layers():
            if not self._get_layer(layer)['visible'] \
                == (layer in [self._get_layers()[0],
                self._get_layers()[-1]]):
                return nemoa.log(
                    'error', """The first and the last
                    layer of a Feedforward network have to be visible,
                    middle layers have to be hidden!""")

        return True
示例#3
0
文件: base.py 项目: Jackieee/nemoa
    def stratifyData(self, algorithm = 'auto'):
        """Stratify data.

        Keyword arguments:
            algorithm -- name of algorithm used for stratification
                'none':
                    probabilities of sources are
                    number of all samples / number of samples in source
                'auto':
                    probabilities of sources are hierarchical distributed
                    as defined in the configuration
                'equal':
                    probabilities of sources are
                    1 / number of sources"""
        nemoa.log('stratify data using \'%s\'' % (algorithm))

        if algorithm.lower() in ['none']:
            allSize = 0
            for src in self.data: allSize += self.data[src]['array'].shape[0]
            for src in self.data: self.data[src]['fraction'] = \
                float(allSize) / float(self.data[src]['array'].shape[0])
            return True
        if algorithm.lower() in ['auto']: return True
        if algorithm.lower() in ['equal']:
            frac = 1.0 / float(len(self.data))
            for src in self.data: self.data[src]['fraction'] = frac
            return True
        return False
示例#4
0
    def __importModelFromFile(self, file):
        """Return new model instance and set configuration and parameters from file."""

        nemoa.log('import model from file')
        nemoa.setLog(indent = '+1')

        # check file
        if not os.path.exists(file):
            if os.path.exists(
                nemoa.workspace.path('models') + file + '.mp'):
                file = nemoa.workspace.path('models') + file + '.mp'
            else: return nemoa.log('error', """
                could not load model '%s':
                file does not exist.""" % file)

        # load model parameters and configuration from file
        nemoa.log('load model: \'%s\'' % file)
        modelDict = nemoa.common.dictFromFile(file)

        model = self.__getModelInstance(
            name    = modelDict['config']['name'],
            config  = modelDict['config'],
            dataset = modelDict['dataset']['cfg'],
            network = modelDict['network']['cfg'],
            system  = modelDict['system']['config'])

        if nemoa.type.isModel(model): model._set(modelDict)
        else: return None 

        nemoa.setLog(indent = '-1')
        return model
示例#5
0
    def get(self,
            type=None,
            name=None,
            merge=['params'],
            params=None,
            id=None):
        """Return configuration as dictionary for given object."""
        if not type in self.__store.keys():
            return nemoa.log(
                'warning', """
            could not get configuration:
            object class '%s' is not known """ % type)

        # search 'name' or 'id' in 'section' or take first entry
        cfg = None

        # get configuration from type and name
        if name:
            if not name in self.__store[type].keys(): return False
            cfg = self.__store[type][name].copy()

        # get configuration from type and id
        elif id:
            for name in self.__store[type].keys():
                if not self.__store[type][name]['id'] == id: continue
                cfg = self.__store[type][name]
            if cfg == None:
                return nemoa.log(
                    'warning', """
                could not get configuration:
                no %s with id %i could be found """ % (type, id))

        # could not identify configuration
        else:
            return nemoa.log(
                'warning',
                "could not get configuration: 'id' or 'name' of object is needed!"
            )

        if not cfg: return None

        # optionaly merge sub dictionaries
        # defined by a list of keys and a dictionary
        if params == None \
            or not isinstance(params, dict)\
            or not nemoa.common.isList(merge):
            return cfg
        subMerge = cfg
        for key in merge:
            if not isinstance(subMerge, dict):
                return cfg
            if not key in subMerge.keys():
                subMerge[key] = {}
            subMerge = subMerge[key]
        for key in params.keys():
            subMerge[key] = params[key]
            cfg['id'] += self.__getObjIDByName('.'.join(merge) + '.' + key,
                                               params[key])

        return cfg
示例#6
0
    def __importModelFromFile(self, file):
        """Return new model instance and set configuration and parameters from file."""

        nemoa.log('import model from file')
        nemoa.setLog(indent='+1')

        # check file
        if not os.path.exists(file):
            if os.path.exists(nemoa.workspace.path('models') + file + '.mp'):
                file = nemoa.workspace.path('models') + file + '.mp'
            else:
                return nemoa.log(
                    'error', """
                could not load model '%s':
                file does not exist.""" % file)

        # load model parameters and configuration from file
        nemoa.log('load model: \'%s\'' % file)
        modelDict = nemoa.common.dictFromFile(file)

        model = self.__getModelInstance(name=modelDict['config']['name'],
                                        config=modelDict['config'],
                                        dataset=modelDict['dataset']['cfg'],
                                        network=modelDict['network']['cfg'],
                                        system=modelDict['system']['config'])

        if nemoa.type.isModel(model): model._set(modelDict)
        else: return None

        nemoa.setLog(indent='-1')
        return model
示例#7
0
def save(network, path=None, filetype=None, plot=None, **kwargs):

    # test if filetype is supported
    if not filetype in filetypes():
        return nemoa.log(
            'error', """could not create plot:
            filetype '%s' is not supported.""" % filetype)

    # get class for plotting from attribute 'plot'
    if not plot: plot = 'graph'
    class_name = plot.lower().title()
    module_name = save.__module__
    try:
        module = importlib.import_module(module_name)
        if not hasattr(module, class_name): raise ImportError()
    except ImportError:
        return nemoa.log(
            'error', """could not plot network '%s':
            plot type '%s' is not supported.""" % (network.name, plot))

    # create plot instance
    plot = getattr(module, class_name)(**kwargs)

    # create plot and save to file
    if plot.create(network): plot.save(path)

    # clear figures and release memory
    plot.release()

    return path
示例#8
0
文件: base.py 项目: Jackieee/nemoa
    def _isNetworkDBNCompatible(self, network=None):
        """Check if a network is compatible to deep beliefe networks.

        Keyword Arguments:
            network -- nemoa network instance

        Description:
            Return True if the following conditions are satisfied:
            (1) The network is MPL compatible
            (2) The network contains an odd number of layers
            (3) The hidden layers are symmetric to the central layer
                related to their number of nodes"""
        if not nemoa.type.isNetwork(network):
            return nemoa.log("error", "could not test network: invalid network instance given!")
        if not self._isNetworkMLPCompatible(network):
            return False
        if not len(network.layers()) % 2 == 1:
            return nemoa.log("error", "DBN / Autoencoder networks expect an odd number of layers!")

        layers = network.layers()
        size = len(layers)
        for id in range(1, (size - 1) / 2):
            symmetric = len(network.layer(layers[id])["nodes"]) == len(network.layer(layers[-id - 1])["nodes"])
            if not symmetric:
                return nemoa.log(
                    "error",
                    """DBN / Autoencoder networks expect a symmetric
                number of hidden nodes, related to their central layer!""",
                )
        return True
示例#9
0
文件: base.py 项目: Jackieee/nemoa
    def getFormatedData(self, data, cols = '*', output = 'array'):
        """Return data in given format.

        Keyword Arguments:
            cols -- name of column group
                default: value '*' does not filter columns
            format"""
        
        # check columns
        if cols == '*': cols = self.getColLabels()
        elif not len(cols) == len(set(cols)): return nemoa.log('error',
            'could not retrieve data: columns are not unique!')
        elif [c for c in cols if c not in self.getColLabels()]: \
            return nemoa.log('error',
            'could not retrieve data: unknown columns!')
    
        # check format
        if isinstance(output, str): fmtTuple = (output, )
        elif isinstance(output, tuple): fmtTuple = output
        else: return nemoa.log('error',
            "could not retrieve data: invalid 'format' argument!")

        # format data
        retTuple = ()
        for fmtStr in fmtTuple:
            if fmtStr == 'array': retTuple += (
                data[cols].view('<f8').reshape(data.size, len(cols)), )
            elif fmtStr == 'recarray': retTuple += (
                data[['label'] + cols], )
            elif fmtStr == 'cols': retTuple += (
                [col.split(':')[1] for col in cols], )
            elif fmtStr in ['rows', 'list']: retTuple += (
                data['label'].tolist(), )
        if isinstance(output, str): return retTuple[0]
        return retTuple
示例#10
0
文件: base.py 项目: Jackieee/nemoa
    def _set(self, dict):
        """Set configuration, parameters and data of model from given dictionary
        return true if no error occured"""

        # get config from dict
        config = self.importConfigFromDict(dict)

        # check self
        if not nemoa.type.isDataset(self.dataset): return nemoa.log('error',
            'could not configure dataset: model does not contain dataset instance!')
        if not nemoa.type.isNetwork(self.network): return nemoa.log('error',
            'could not configure network: model does not contain network instance!')
        if not nemoa.type.isSystem(self.system): return nemoa.log('error',
            'could not configure system: model does not contain system instance!')

        self.__config = config['config'].copy()
        self.network._set(**config['network'])
        self.dataset._set(**config['dataset'])

        ## prepare
        if not 'update' in config['system']['config']:
            config['system']['config']['update'] = {'A': False}

        ## 2do system._set(...) shall create system
        ## and do something like self.configure ...

        # create system
        self.system = nemoa.system.new(
            config  = config['system']['config'].copy(),
            network = self.network,
            dataset = self.dataset )
        self.system._set(**config['system'])

        return self
示例#11
0
文件: config.py 项目: Jackieee/nemoa
    def __addObjToStore(self, objConf):
        """link object configuration to object dictionary."""
        if not isinstance(objConf, dict):
            return False
        
        type = objConf['class']
        name = objConf['name']
        config = objConf['config']

        nemoa.log('adding %s: %s' % (type, name))

        key = None
        objID = 0

        if not type in self.__store.keys():
            nemoa.log('error', 'could not register object \'%s\': not supported object type \'%s\'!' % (name, type))
            return False

        key = self.__getNewKey(self.__store[type], name)
        objID = self.__getObjIDByName(type, key)

        # add configuration to object tree
        self.__store[type][key] = config
        self.__store[type][key]['id'] = objID

        # add entry to index
        self.__index[objID] = {'type': type, 'name': key, 'project': objConf['project']}

        return objID
示例#12
0
文件: base.py 项目: Goatbuster/nemoa
    def _get_default(self, key=None, *args, **kwargs):
        """Get default value."""

        import copy

        if not key: return copy.deepcopy(self._config['default'])
        elif not key in self._config['default']:
            return nemoa.log(
                'error', """could not get default value:
                key '%s' is not valid.""" % key)
        retval = self._config['default'][key]
        parent = key
        while args:
            if not isinstance(args[0], str) \
                or not args[0] in retval:
                return nemoa.log(
                    'error', """could not get default
                    value: '%s' does not contain key '%s'.""" %
                    (parent, args[0]))
            parent = args[0]
            retval = retval[args[0]]
            args = args[1:]

        if isinstance(retval, dict): return copy.deepcopy(retval)
        return retval
示例#13
0
文件: config.py 项目: Jackieee/nemoa
    def __importConfigFile(self, file):
        """Import configuration (.ini) file."""

        # search definition file
        if os.path.isfile(file):
            configFile = file
        elif os.path.isfile(self.__basepath['workspace'] + file):
            configFile = self.__basepath['workspace'] + file
        else:
            nemoa.log('warning', "configuration file '%s' does not exist!" % (file))
            return False

        # logger info
        nemoa.log("parsing configuration file: '" + configFile + "'")
        nemoa.setLog(indent = '+1')

        # import and register objects without testing
        importer = configFileImporter(self)
        objConfList = importer.load(configFile)

        for objConf in objConfList:
            self.__addObjToStore(objConf)

        self.__check(objConfList)

        nemoa.setLog(indent = '-1')
        return True
示例#14
0
文件: base.py 项目: Jackieee/nemoa
    def _isNetworkMLPCompatible(self, network=None):
        """Check if a network is compatible to multilayer perceptrons.

        Keyword Arguments:
            network -- nemoa network instance

        Description:
            Return True if the following conditions are satisfied:
            (1) The network contains at least three layers
            (2) All layers of the network are not empty
            (3) The first and last layer of the network are visible,
                all middle layers of the network are hidden"""
        if not nemoa.type.isNetwork(network):
            return nemoa.log("error", "could not test network: invalid network instance given!")
        if len(network.layers()) < 3:
            return nemoa.log("error", "Multilayer networks need at least three layers!")
        for layer in network.layers():
            if not len(network.layer(layer)["nodes"]) > 0:
                return nemoa.log("error", "Feedforward networks do not allow empty layers!")
            if not network.layer(layer)["visible"] == (layer in [network.layers()[0], network.layers()[-1]]):
                return nemoa.log(
                    "error",
                    """The first and the last layer
                    of a multilayer feedforward network have to be visible,
                    middle layers have to be hidden!""",
                )
        return True
示例#15
0
文件: graph.py 项目: Goatbuster/nemoa
def _graph_decode(graph):

    # no decoding
    if not 'coding' in graph.graph \
        or not graph.graph['coding'] \
        or graph.graph['coding'].lower() == 'none':
        return graph

    # base64 decoding
    elif graph.graph['coding'] == 'base64':
        graph.graph['params'] = nemoa.common.compress.loads(
            graph.graph['params'], encode='base64')

        for node in graph.nodes():
            graph.node[node]['params'] = nemoa.common.compress.loads(
                graph.node[node]['params'], encode='base64')

        for edge in graph.edges():
            graph.edges[edge]['params'] = nemoa.common.compress.loads(
                graph.edges[edge]['params'], encode='base64')

        graph.graph['coding'] == 'none'
        return graph

    else:
        nemoa.log(
            'error', """could not decode graph parameters:
            unsupported coding '%s'.""" % coding)

    return {}
示例#16
0
文件: base.py 项目: Jackieee/nemoa
    def csvGetData(self, name):
        conf    = self.cfg['table'][name]['source']
        file    = conf['file']
        delim   = conf['delimiter'] if 'delimiter' in conf \
            else nemoa.common.csvGetDelimiter(file)
        cols    = conf['usecols']
        names   = tuple(self.getColLabels())
        formats = tuple(['<f8' for x in names])
        if not 'rows' in conf or conf['rows']:
            cols = (0,) + cols
            names = ('label',) + names
            formats = ('<U12',) + formats
        dtype = {'names': names, 'formats': formats}

        nemoa.log("import data from csv file: " + file)

        try:
            #data = numpy.genfromtxt(file, skiprows = 1, delimiter = delim,
                #usecols = cols, dtype = dtype)
            data = numpy.loadtxt(file, skiprows = 1, delimiter = delim,
                usecols = cols, dtype = dtype)
        except:
            nemoa.log('error', 'could not import data from file!')
            return None

        return data
示例#17
0
文件: base.py 项目: Goatbuster/nemoa
    def _is_compatible_dbn(self):
        """Test compatibility to deep beliefe networks.

        Returns:
            Boolean value which is True if the following conditions are
            satisfied:
            (1) The network is compatible to multilayer feedforward
                networks: see function _is_compatible_mlff
            (2) The network contains an odd number of layers
            (3) The hidden layers are symmetric to the central middle
                layer related to their number of nodes

        """

        # test if network is MLFF compatible
        if not self._is_compatible_mlff(): return False

        # test if the network contains odd number of layers
        if not len(self._get_layers()) % 2 == 1:
            return nemoa.log(
                'error', """DBNs expect an odd
                number of layers.""")

        # test if the hidden layers are symmetric
        layers = self._get_layers()
        size = len(layers)
        for lid in range(1, (size - 1) / 2):
            symmetric = len(self._get_layer(layers[lid])['nodes']) \
                == len(self._get_layer(layers[-lid-1])['nodes'])
            if not symmetric:
                return nemoa.log(
                    'error', """DBNs expect a symmetric number of hidden
                nodes, related to their central middle layer!""")

        return True
示例#18
0
文件: base.py 项目: Jackieee/nemoa
    def __confDataset(self, dataset = None, network = None, **kwargs):
        """Configure model.dataset to given dataset and network.

        Keyword Arguments:
            dataset -- dataset instance
            network -- network instance
        """
        dataset = self.dataset
        network = self.network

        # link dataset instance
        if nemoa.type.isDataset(dataset):
            self.dataset = dataset

        # check if dataset instance is valid
        if not nemoa.type.isDataset(self.dataset):
            nemoa.log('error',
            'could not configure dataset: no dataset instance available!')
            return False

        # check if dataset is empty
        if self.dataset.isEmpty():
            return True

        # prepare params
        if not network and not self.network:
            nemoa.log('error',
            'could not configure dataset: no network instance available!')
            return False

        return self.dataset.configure(network = network \
            if not network == None else self.network)
示例#19
0
    def initialize(self):
        """Initialize model parameters."""

        if not nemoa.common.type.isdataset(self.dataset):
            return nemoa.log(
                'error', """could not initialize model:
                dataset is not valid.""")
        if not nemoa.common.type.isnetwork(self.network):
            return nemoa.log(
                'error', """could not initialize model:
                network is not valid.""")
        if not nemoa.common.type.issystem(self.system):
            return nemoa.log(
                'error', """could not initialize model:
                system is not valid.""")

        retval = True

        # initialize dataset to system including normalization
        retval &= bool(self.dataset.initialize(self.system))
        # initialize system parameters by using statistics from dataset
        retval &= bool(self.system.initialize(self.dataset))
        # initialize network parameters with system parameters
        retval &= bool(self.network.initialize(self.system))

        return retval
示例#20
0
    def getUnitRelation(self, dataset, stat = 10000, units = None,
        relation = 'correlation()', format = 'array', **kwargs):

        # get mapping, data and units
        mapping = self.getMapping()
        data    = dataset.getData(cols = (mapping[0], mapping[-1]))
        units   = self._assertUnitTuple(units)

        # get method and method specific parameters
        method, ukwargs = nemoa.common.strSplitParams(relation)
        params = nemoa.common.dictMerge(ukwargs, kwargs)

        # get relation as numpy array
        if method == 'correlation': M = self.getUnitCorrelation(\
            units = units, data = data, **params)
        elif method == 'interaction': M = self.getUnitInteraction(\
            units = units, data = data, mapping = mapping, **params)
        elif method == 'knockout': M = self.getUnitKnockout(\
            units = units, data = data, mapping = mapping, **params)
        else: return nemoa.log('error',
            "could not evaluate unit relations: unknown relation '%s'" % (method))

        # transform matrix
        if 'transform' in params:
            if 'C' in params['transform']:
                C = self.getUnitCorrelationMatrix(units = units, data = data)
            try:
                T = eval(params['transform'])
                M = T
            except: return nemoa.log('error',
                'could not transform unit relation matrix: invalid syntax!')

        # create formated output
        if format == 'array': return M
        if format == 'dict': return nemoa.common.dictFromArray(M, units)
示例#21
0
    def loadProject(self, project):
        """Import configuration files from user project."""

        nemoa.log('import private resources')
        nemoa.setLog(indent='+1')

        # check if project exists
        if not project in self.__listUserWorkspaces():
            nemoa.log(
                'warning', """
                could not open project '%s':
                project folder could not be found in '%s'!
                """ % (project, self.__basepath['user']))
            return False

        # set project
        self.__project = project

        # update paths
        self.__updatePaths(base='user')

        # update path of cache
        self.__updateCachePaths()

        # update logger to current logfile
        nemoa.initLogger(logfile=self.__path['logfile'])

        # import object configurations for current project
        self.__scanForConfigFiles()
        self.__scanForScripts()
        self.__scanForNetworks()

        nemoa.setLog(indent='-1')
        return True
示例#22
0
    def _isNetworkDBNCompatible(self, network = None):
        """Check if a network is compatible to deep beliefe networks.

        Keyword Arguments:
            network -- nemoa network instance

        Description:
            Return True if the following conditions are satisfied:
            (1) The network is MPL compatible
            (2) The network contains an odd number of layers
            (3) The hidden layers are symmetric to the central layer
                related to their number of nodes"""
        if not nemoa.type.isNetwork(network): return nemoa.log('error',
            'could not test network: invalid network instance given!')
        if not self._isNetworkMLPCompatible(network): return False
        if not len(network.layers()) % 2 == 1: return nemoa.log('error',
            'DBN / Autoencoder networks expect an odd number of layers!')

        layers = network.layers()
        size = len(layers)
        for id in range(1, (size - 1) / 2):
            symmetric = len(network.layer(layers[id])['nodes']) \
                == len(network.layer(layers[-id-1])['nodes'])
            if not symmetric: return nemoa.log('error',
                """DBN / Autoencoder networks expect a symmetric
                number of hidden nodes, related to their central layer!""")
        return True
示例#23
0
    def _isDatasetGaussNormalized(self, dataset = None):
        """Test if a dataset contains gauss normalized data.

        Keyword Arguments:
            dataset -- nemoa dataset instance

        Description:
            Return True if the following conditions are satisfied:
            (1) The absolute mean value of a given number of random samples
                of the dataset is below a given maximum (default 0.05)
            (2) The standard deviation of a given number of random samples
                of the dataset is below a given maximum (default 1.05)"""

        if not nemoa.type.isDataset(dataset): return nemoa.log('error',
            'could not test dataset: invalid dataset instance given!')

        size    = 100000 # number of samples
        maxMean = 0.05   # allowed maximum for absolute mean value
        maxSdev = 1.05   # allowed maximum for standard deviation

        data = dataset.getData(size)

        mean = data.mean()
        if numpy.abs(mean) >= maxMean: return nemoa.log('error', """
            The dataset does not contain gauss normalized data:
            The mean value is %.3f!""" % (mean))

        sdev = data.std()
        if sdev >= maxSdev: return nemoa.log('error', """
            The dataset does not contain gauss normalized data:
            The standard deviation is %.3f!""" % (sdev))

        return True
示例#24
0
文件: image.py 项目: Goatbuster/nemoa
def get_plot(dataset, func=None, plot=None, **kwargs):

    import importlib

    # get evaluation function
    fname = func or 'sample'
    fdict = dataset.get('algorithm', fname)
    if not isinstance(fdict, dict):
        return nemoa.log(
            'error', "could not plot dataset '%s': "
            "dataset evaluation function '%s' "
            "is not supported." % (dataset.name, fname))

    # get plot type
    plot = plot or fdict.get('plot', None) or 'histogram'

    # get plot class and module name
    cname = plot.lower().title()
    mname = save.__module__
    try:
        module = importlib.import_module(mname)
        if not hasattr(module, cname): raise ImportError()
    except ImportError:
        return nemoa.log(
            'error', "could not plot dataset '%s': "
            "plot type '%s' is not supported." % (dataset.name, plot))

    # create plot
    plot = getattr(module, cname)(func=fname, **kwargs)
    if plot.create(dataset): return plot

    plot.release()
    return None
示例#25
0
    def _isNetworkMLPCompatible(self, network = None):
        """Check if a network is compatible to multilayer perceptrons.

        Keyword Arguments:
            network -- nemoa network instance

        Description:
            Return True if the following conditions are satisfied:
            (1) The network contains at least three layers
            (2) All layers of the network are not empty
            (3) The first and last layer of the network are visible,
                all middle layers of the network are hidden"""
        if not nemoa.type.isNetwork(network): return nemoa.log('error',
            'could not test network: invalid network instance given!')
        if len(network.layers()) < 3: return nemoa.log('error',
            'Multilayer networks need at least three layers!')
        for layer in network.layers():
            if not len(network.layer(layer)['nodes']) > 0: return nemoa.log(
                'error', 'Feedforward networks do not allow empty layers!')
            if not network.layer(layer)['visible'] \
                == (layer in [network.layers()[0], network.layers()[-1]]):
                return nemoa.log('error', """The first and the last layer
                    of a multilayer feedforward network have to be visible,
                    middle layers have to be hidden!""")
        return True
示例#26
0
文件: config.py 项目: Jackieee/nemoa
    def loadProject(self, project):
        """Import configuration files from user project."""

        nemoa.log('import private resources')
        nemoa.setLog(indent = '+1')

        # check if project exists
        if not project in self.__listUserWorkspaces():
            nemoa.log('warning', """
                could not open project '%s':
                project folder could not be found in '%s'!
                """ % (project, self.__basepath['user']))
            return False
        
        # set project
        self.__project = project

        # update paths
        self.__updatePaths(base = 'user')

        # update path of cache
        self.__updateCachePaths()

        # update logger to current logfile
        nemoa.initLogger(logfile = self.__path['logfile'])

        # import object configurations for current project
        self.__scanForConfigFiles()
        self.__scanForScripts()
        self.__scanForNetworks()

        nemoa.setLog(indent = '-1')
        return True
示例#27
0
    def __checkSystemConf(self, objConf):
        """Check and update system configuration"""
        type = objConf['class']
        name = objConf['name']
        conf = objConf['config']

        if not 'package' in conf:
            return nemoa.log(
                'warning',
                "skipping system '" + name + "': missing parameter 'package'!")
        if not 'class' in conf:
            return nemoa.log(
                'warning',
                "skipping system '" + name + "': missing parameter 'class'!")

        conf['description'] = conf['description'].replace('\n', '') \
            if 'description' in conf else conf['name']

        # check if system exists
        try:
            exec "import nemoa.system." + conf['package']
        except:
            return nemoa.log(
                'warning',
                "skipping system '%s': package 'system.%s' could not be found!"
                % (name, conf['package']))
        return objConf
示例#28
0
文件: base.py 项目: Jackieee/nemoa
    def normalizeData(self, algorithm = 'gauss'):
        """Normalize stratified data

        Keyword arguments:
            algorithm -- name of algorithm used for data normalization
                'gauss', 'z-trans':
                    Gaussian normalization (aka z-transformation)"""

        nemoa.log('normalize data using \'%s\'' % (algorithm))

        if algorithm.lower() in ['gauss', 'z-trans']:

            # get data for calculation of mean and variance
            # for single source datasets take all data
            # for multi source datasets take a big bunch of stratified data
            if len(self.data.keys()) > 1: data = \
                self.getData(size = 1000000, output = 'recarray')
            else: data = self.getSingleSourceData(source = self.data.keys()[0])

            # iterative update sources
            # get mean and standard deviation per column (recarray)
            # and update the values
            for src in self.data:
                if self.data[src]['array'] == None: continue
                for col in self.data[src]['array'].dtype.names[1:]:
                    self.data[src]['array'][col] = \
                        (self.data[src]['array'][col] - data[col].mean()) \
                        / data[col].std()
            return True

        return False
示例#29
0
文件: text.py 项目: Goatbuster/nemoa
    def load(self, path):
        """Return network configuration as dictionary.

        Args:
            path: configuration file used to generate network
                configuration dictionary.

        """

        structure = {'network': {'type': 'str'}}
        network = nemoa.common.inifile.load(path, structure)
        if not network \
            or not 'network' in network \
            or not 'type' in network['network']:
            return nemoa.log(
                'error', """could not import network:
                configuration file '%s' is not valid.""" % path)

        if network['network']['type'] in \
            ['layer.MultiLayer', 'layer.Shallow', 'layer.Factor']:
            return self._parse_layer_network(path)

        return nemoa.log(
            'error', """could not import network
            configuration file '%s' contains unsupported network
            type '%s'.""" % (path, network['network']['type']))
示例#30
0
    def __createNewModel(self,
                         name,
                         config=None,
                         dataset=None,
                         network=None,
                         system=None,
                         configure=True,
                         initialize=True):
        nemoa.log('create new model')
        nemoa.setLog(indent='+1')

        model = self.__getModelInstance(name=name,
                                        config=config,
                                        dataset=dataset,
                                        network=network,
                                        system=system)

        if not nemoa.type.isModel(model):
            nemoa.log('error', 'could not create new model!')
            nemoa.setLog(indent='-1')
            return False

        # configure model
        if configure: model.configure()

        # initialize model parameters
        if initialize: model.initialize()

        nemoa.setLog(indent='-1')
        return model
示例#31
0
文件: gene.py 项目: Goatbuster/nemoa
    def _install_pkg(self, pkg=None):
        if not pkg:
            nemoa.log('note', """trying to install
            bioconductor base""")
        else:
            nemoa.log(
                'note', """trying to install
               bioconductor package: '%s'""" % pkg)

        # try to evaluate the remote R script biocLite()
        bioclite = "https://bioconductor.org/biocLite.R"
        sysstout = sys.stdout
        try:
            sys.stdout = NullDevice()
            from rpy2.robjects.packages import importr
            base = importr('base')
            base.source(bioclite)
            base.require('biocLite')
            sys.stdout = sysstout
        except:
            sys.stdout = sysstout
            return nemoa.log(
                'error', """could not evaluate remote R
                script: '%s'""" % bioclite)

        # try to install bioconductor packages with biocLite()
        if not pkg: return self._exec_rcmd("biocLite()")
        return self._exec_rcmd("biocLite('%s')" % pkg)
示例#32
0
def convert(list, input, output = None, filter = False, quiet = True):

    genericClasses = ['number', 'string', 'float']

    if isinstance(list, (numpy.ndarray)):
        list = list.tolist()
        inputDtype = 'nparray'
    else:
        inputDtype = 'list'

    # 'input'
    if input in genericClasses:
        inputClass  = 'generic'
        inputFormat = input
    elif ':' in input:
        inputClass  = input.lower().split(':')[0].strip()
        inputFormat = input.lower().split(':')[1].strip()
    else:
        nemoa.log('warning', 'could not convert list: unknown input format "' + input + '"!')
        return None

    # 'output'
    if output in genericClasses:
        outputClass  = 'generic'
        outputFormat = output
    elif output == None:
        outputClass  = inputClass
        outputFormat = None
    elif ':' in input: 
        outputClass  = output.lower().split(':')[0].strip()
        outputFormat = output.lower().split(':')[1].strip()
    else:
        mp_warnung('could not convert list: unknown output format "' + output + '"!')
        return None

    # 'input' vs 'output'
    if inputClass != outputClass:
        nemoa.log('warning', '"' + inputClass + '" can not be converted to "' + outputClass + '"')
        return None

    # trivial cases
    if inputClass == 'generic' or inputFormat == outputFormat:
        if inputDtype == 'nparray':
            return numpy.asarray(list), numpy.asarray([])
        else:
            return list, []

    # import annotation module
    modName = inputClass.lower()
    import importlib
    module = importlib.import_module('nemoa.annotation.' + modName)
    converter = getattr(module, modName)()
    
    outList, outLost = converter.convert_list(list, inputFormat, outputFormat, filter, quiet = quiet)

    if inputDtype == 'nparray':
        return numpy.asarray(outList), numpy.asarray(outLost)

    return outList, outLost
示例#33
0
def interactive():
    try:
        call(['ipython', 'nemoa.interactive.py', '-i'])
    except:
        nemoa.log(
            'error', """
            could not start interactive nemoa shell:
            you have to install ipython!""")
示例#34
0
def getdelim(path, delimiters = [',', ';', '\t', ' '],
    minprobesize = 3, maxprobesize = 100):
    """Get delimiter from CSV file.

    Args:
        path (string): file path to CSV file.
        delimiters (list, optional): list of strings containing
            delimiter candidates to search for.
        minprobesize (integer, optional): minimum number of non comment,
            non empty lines used to detect csv delimiter.
        maxprobesize (integer, optional): maximum number of non comment,
            non empty lines used to detect csv delimiter.

    Returns:
        String containing delimiter of CSV file or False if delimiter
        could not be detected.

    """

    # check file
    if not os.path.isfile(path):
        return nemoa.log('error', """could not determine delimiter:
            file '%s' does not exist.""" % path)

    delimiter = None
    with open(path, 'r') as csvfile:
        lines = 1
        probe = ''
        for line in csvfile:

            # check termination criteria
            if bool(delimiter) or lines > maxprobesize:
                break

            # check exclusion criteria
            sline = line.lstrip(' ')
            if sline.startswith('#') or sline in ['\n', '\r\n']:
                continue

            # increase probe
            probe += line
            lines += 1

            # try to detect delimiter of probe
            if lines > minprobesize:
                try:
                    dialect = csv.Sniffer().sniff(probe, delimiters)
                except:
                    continue
                delimiter = dialect.delimiter

    if not delimiter:
        return nemoa.log('warning', """could not determine delimiter
            of csv file '%s'!""" % path)

    return delimiter
示例#35
0
def interactive():
    try:
        call(["ipython", "nemoa.interactive.py", "-i"])
    except:
        nemoa.log(
            "error",
            """
            could not start interactive nemoa shell:
            you have to install ipython!""",
        )
示例#36
0
文件: base.py 项目: Goatbuster/nemoa
 def _get_edge(self, edge):
     if not isinstance(edge, tuple):
         return nemoa.log(
             'error', """could not get edge:
             edge '%s' is unkown.""" % (edge))
     if not edge in self._graph.edges:
         return nemoa.log(
             'error', """could not get edge:
             edge ('%s', '%s') is unkown.""" % edge)
     return self._graph.edges[edge]
示例#37
0
    def __checkNetworkConf(self, objConf):
        """Check and update network configuration."""
        type = objConf['class']
        name = objConf['name']
        conf = objConf['config']

        # create 'layer', 'visible' and 'hidden' from 'layers'
        if 'layers' in conf:
            conf['layer'] = []
            conf['visible'] = []
            conf['hidden'] = []
            for layer in conf['layers']:
                if '=' in layer:
                    layerName = layer.split('=')[0].strip()
                    layerType = layer.split('=')[1].strip().lower()
                else:
                    layerName = layer.strip()
                    layerType = 'visible'

                conf['layer'].append(layerName)
                if layerType == 'visible':
                    conf['visible'].append(layerName)
                else:
                    conf['hidden'].append(layerName)
            del conf['layers']

        # get config from source file
        if 'source' in conf:
            if not 'file' in conf['source']:
                return nemoa.log(
                    'warning', "skipping network '" + name + "': "
                    "missing source information! (parameter: 'source:file')")

            file = conf['source']['file']
            if not self.getPath(file, check=True):
                return nemoa.log(
                    'warning', "skipping network '%s': file '%s' not found!" %
                    (name, file))

            objConf['config']['source']['file'] = self.getPath(file)

            if not 'file_format' in conf['source']:
                objConf['config']['source'][
                    'file_format'] = nemoa.common.getFileExt(file)

            format = objConf['config']['source']['file_format']

            # get network config
            networkCfg = self.__registerNetwork(file, format)
            if not networkCfg:
                return nemoa.log('warning', "skipping network '%s'" % (name))
            for key in networkCfg:
                objConf['config'][key] = networkCfg[key]

        return objConf
示例#38
0
文件: base.py 项目: Jackieee/nemoa
    def __triggerKeyEvent(self):
        """Check Keyboard."""

        c = nemoa.common.getch()
        if isinstance(c, str):
            if c == "q":
                nemoa.log("note", "... aborting optimization")
                self.__system._config["optimize"]["updates"] = self.__state["epoch"]
                self.__state["abort"] = True

        return True
示例#39
0
文件: gene.py 项目: Jackieee/nemoa
    def __init__(self):

        stdout = sys.stdout
        try:
            sys.stdout = NullDevice()
            import rpy2.robjects
            self.robjects = rpy2.robjects
            sys.stdout = stdout
        except:
            sys.stdout = stdout
            nemoa.log('error', "could not import python package rpy2!")
示例#40
0
文件: config.py 项目: Jackieee/nemoa
    def __getObjByID(self, id):
        """Return object from store by id."""
        if not id in self.__index:
            nemoa.log('warning', '2DO')
            return None

        oClass = self.__index[id]['type']
        oName  = self.__index[id]['name']
        oPrj   = self.__index[id]['project']
        oConf  = self.__store[oClass][oName].copy()

        return {'class': oClass, 'name': oName, 'project': oPrj, 'config':  oConf}
示例#41
0
文件: base.py 项目: Jackieee/nemoa
    def optimize(self, schedule = None, **kwargs):
        """Optimize system parameters."""

        nemoa.log('optimize model')
        nemoa.setLog(indent = '+1')

        # check if model is empty
        if self.isEmpty():
            nemoa.log('warning', "empty models can not be optimized!")
            nemoa.setLog(indent = '-1')
            return self

        # check if model is configured
        if not self.__isConfigured():
            nemoa.log('error',
                'could not optimize model: model is not yet configured!')
            nemoa.setLog(indent = '-1')
            return False

        # get optimization schedule
        if schedule == None: schedule = self.system.getType() + '.default'
        elif not '.' in schedule: schedule = \
            self.system.getType() + '.' + schedule
        schedule = nemoa.workspace.getConfig(
            type = 'schedule', config = schedule,
            merge = ['params', self.system.getType()],
            **kwargs)
        if not schedule:
            nemoa.log('error', """
                could not optimize system parameters:
                optimization schedule is not valid!""")
            nemoa.setLog(indent = '-1')
            return self
        
        # optimization of system parameters
        nemoa.log("starting optimization schedule: '%s'" % (schedule['name']))
        nemoa.setLog(indent = '+1')

        # 2DO: find better solution for multistage optimization
        if 'stage' in schedule and len(schedule['stage']) > 0:
            for stage, params in enumerate(config['stage']):
                self.system.optimizeParams(self.dataset, **params)
        elif 'params' in schedule:
            self.system.optimizeParams(
                dataset = self.dataset, schedule = schedule)

        nemoa.setLog(indent = '-1')

        # update network
        self.system.updateNetwork(self.network)

        nemoa.setLog(indent = '-1')
        return self
示例#42
0
    def __triggerKeyEvent(self):
        """Check Keyboard."""

        c = nemoa.common.getch()
        if isinstance(c, str):
            if c == 'q':
                nemoa.log('note', '... aborting optimization')
                self.__system._config['optimize']['updates'] = \
                    self.__state['epoch']
                self.__state['abort'] = True

        return True
示例#43
0
def convert(list, input, output = None, filter = False):

    generic_types = ['number', 'string', 'float']

    if isinstance(list, (numpy.ndarray)):
        list = list.tolist()
        input_dtype = 'nparray'
    else: input_dtype = 'list'

    # 'input'
    if input in generic_types:
        input_class  = 'generic'
        input_format = input
    elif ':' in input:
        input_class  = input.lower().split(':')[0].strip()
        input_format = input.lower().split(':')[1].strip()
    else: return nemoa.log('warning', """could not convert list:
        unknown input format '%s'.""" % input)

    # 'output'
    if output in generic_types:
        output_class  = 'generic'
        output_format = output
    elif not output:
        output_class  = input_class
        output_format = None
    elif ':' in input:
        output_class  = output.lower().split(':')[0].strip()
        output_format = output.lower().split(':')[1].strip()
    else: return nemoa.log('warning', """could not convert list:
        unknown output format '%s'.""" % output)

    # 'input' vs 'output'
    if input_class != output_class:
        return nemoa.log('warning', "'%s' can not be converted to '%s'"
            % (input_class, output_class))

    # trivial cases
    if input_class == 'generic' or input_format == output_format:
        if input_dtype == 'nparray':
            return numpy.asarray(list), numpy.asarray([])
        else: return list, []

    # import annotation module
    module_name = input_class.lower()
    module = importlib.import_module('nemoa.dataset.commons.labels.' + module_name)
    converter = getattr(module, module_name)()
    output_list, output_lost = converter.convert_list(
        list, input_format, output_format, filter)
    if input_dtype == 'nparray':
        return numpy.asarray(output_list), numpy.asarray(output_lost)
    return output_list, output_lost
示例#44
0
def getlabelcolumn(path, delimiter = None):
    """Get column id for column containing row labels from CSV file.

    Args:
        path (string): file path to CSV file.
        delimiter (string, optional): string containing CSV delimiter.

    Returns:
        Integer containing first index of CSV data column containing
        strings.

    """

    # check file
    if not os.path.isfile(path):
        return nemoa.log('error', """could not get csv row label column
            id: file '%s' does not exist.""" % path)

    # get delimiter
    if not delimiter:
        delimiter = getdelim(path)
    if not delimiter:
        return nemoa.log('error', """could not get csv row label column
            id: unknown delimiter.""")

    # get first and second non comment and non empty line
    first = None
    second = None
    with open(path, 'r') as csvfile:
        for line in csvfile:

            # check exclusion criteria
            sline = line.lstrip(' ')
            if sline.startswith('#') or sline in ['\n', '\r\n']:
                continue

            if not first: first = line
            elif not second:
                second = line
                break

    if first == None or second == None:
        return nemoa.log('error', """could not get csv row label column
            id: file '%s' is not valid.""" % (path))

    colvals = second.split(delimiter)
    colvals = [col.strip('\"\' \n') for col in colvals]
    for colid, colval in enumerate(colvals):
        try: float(colval)
        except ValueError: return colid

    return False
示例#45
0
文件: base.py 项目: Jackieee/nemoa
    def __initState(self):

        epochTime = time.time()

        nemoa.log("note", "press 'q' if you want to abort the optimization")
        self.__state = {"startTime": epochTime, "epoch": 0, "inspection": None, "abort": False}
        if self.__inspect:
            self.__state["inspectTime"] = epochTime
        if self.__estimate:
            self.__state["estimateStarted"] = False
            self.__state["estimateEnded"] = False

        return True
示例#46
0
    def __getInstance(self, type = None, config = None, empty = False, **kwargs):
        """Return new instance of given object type and configuration."""
        nemoa.log('create%s %s instance' % (' empty' if empty else '', type))
        nemoa.setLog(indent = '+1')

        # import module
        module = importlib.import_module('nemoa.' + str(type))

        # get objects configuration as dictionary
        config = nemoa.workspace.getConfig(type = type,
            config = config, **kwargs)
        if not isinstance(config, dict):
            nemoa.log('error', """could not create %s instance:
                unknown configuration!""" % (type))
            nemoa.setLog(indent = '-1')
            return None

        # create and initialize new instance of given class
        instance = module.empty() if empty \
            else module.new(config = config, **kwargs)

        # check instance class
        if not nemoa.type.isInstanceType(instance, type):
            nemoa.log('error', """could not create %s instance:
                invalid configuration!""" % (type))
            nemoa.setLog(indent = '-1')
            return None

        nemoa.log('name of %s is: \'%s\'' % (type, instance.name()))
        nemoa.setLog(indent = '-1')
        return instance
示例#47
0
文件: config.py 项目: Jackieee/nemoa
    def __checkNetworkConf(self, objConf):
        """Check and update network configuration."""
        type = objConf['class']
        name = objConf['name']
        conf = objConf['config']

        # create 'layer', 'visible' and 'hidden' from 'layers'
        if 'layers' in conf:
            conf['layer']   = []
            conf['visible'] = []
            conf['hidden']  = []
            for layer in conf['layers']:
                if '=' in layer:
                    layerName = layer.split('=')[0].strip()
                    layerType = layer.split('=')[1].strip().lower()
                else:
                    layerName = layer.strip()
                    layerType = 'visible'

                conf['layer'].append(layerName)
                if layerType == 'visible':
                    conf['visible'].append(layerName)
                else:
                    conf['hidden'].append(layerName)
            del conf['layers']

        # get config from source file
        if 'source' in conf:
            if not 'file' in conf['source']: return nemoa.log('warning', 
                "skipping network '" + name + "': "
                "missing source information! (parameter: 'source:file')")

            file = conf['source']['file']
            if not self.getPath(file, check = True): return nemoa.log('warning', 
                "skipping network '%s': file '%s' not found!" % (name, file))

            objConf['config']['source']['file'] = self.getPath(file)

            if not 'file_format' in conf['source']:
                objConf['config']['source']['file_format'] = nemoa.common.getFileExt(file)

            format = objConf['config']['source']['file_format']

            # get network config
            networkCfg = self.__registerNetwork(file, format)
            if not networkCfg: return nemoa.log('warning', 
                "skipping network '%s'" % (name))
            for key in networkCfg: objConf['config'][key] = networkCfg[key]

        return objConf
示例#48
0
文件: base.py 项目: Jackieee/nemoa
    def save_graph(self, file = None, format = 'gml'):
        if file == None:
            nemoa.log('critical', "no save path was given")
            
        # create path if not available
        if not os.path.exists(os.path.dirname(file)):
            os.makedirs(os.path.dirname(file))

        # everythink seems to be fine
        # nemoa.log("saving graph to %s" % (file))
        
        if format == 'gml':
            G = self.graph.copy()
            networkx.write_gml(G, file)
示例#49
0
文件: base.py 项目: Jackieee/nemoa
 def configure(self):
     """Configure model."""
     nemoa.log('configure model \'%s\'' % (self.name()))
     nemoa.setLog(indent = '+1')
     if not 'check' in self.__config:
         self.__config['check'] = {'dataset': False,
             'network': False, 'System': False}
     self.__config['check']['dataset'] = \
         self.dataset.configure(network = self.network)
     if not self.__config['check']['dataset']:
         nemoa.log('error', """could not configure model: dataset
             could not be configured!""")
         nemoa.setLog(indent = '-1')
         return False
     self.__config['check']['network'] = \
         self.network.configure(dataset = self.dataset,
             system = self.system)
     if not self.__config['check']['network']:
         nemoa.log('error', """could not configure model: network
             could not be configured!""")
         nemoa.setLog(indent = '-1')
         return False
     self.__config['check']['system'] = \
         self.system.configure(network = self.network,
             dataset = self.dataset)
     if not self.__config['check']['system']:
         nemoa.log('error', """could not configure model: system
             could not be configured!""")
         nemoa.setLog(indent = '-1')
         return False
     nemoa.setLog(indent = '-1')
     return True
示例#50
0
文件: config.py 项目: Jackieee/nemoa
    def __delObjConf(self, objConf):
        """Unlink object configuration from object dictionary."""
        if not objConf:
            return False
        id = self.__getObjIDByName(objConf['class'], objConf['name'])
        if not id in self.__index.keys():
            nemoa.log('warning', '2do')
            return False

        # delete configuration from tree
        
        
        # delete entry in index
        self.__index.pop(id)
        return True
示例#51
0
文件: base.py 项目: Jackieee/nemoa
    def _assertUnitTuple(self, units=None):
        """Return tuple with lists of valid input and output units."""

        mapping = self.getMapping()
        if units == None:
            return (self.getUnits(group=mapping[0])[0], self.getUnits(group=mapping[-1])[0])
        if isinstance(units[0], str):
            units[0] = self.getUnits(group=units[0])[0]
        elif isinstance(units[0], list):
            pass
        if isinstance(units[1], str):
            units[1] = self.getUnits(group=units[1])[0]
        elif isinstance(units[1], list):
            pass

        elif isinstance(units[0], list) and isinstance(units[1], list):
            # 2do: test if units are valid
            pass
        else:
            return nemoa.log(
                "error",
                """could not evaluate unit relations:
            parameter units has invalid format!""",
            )

        return units
示例#52
0
文件: base.py 项目: Jackieee/nemoa
    def getDataMean(data, norm="M"):
        """Return mean of data.

        Keyword Arguments:
            data -- numpy array containing data
            norm -- data mean norm
                'M': Arithmetic Mean of Values
                'ME': Mean of Errors
                'MSE': Mean of Squared Errors
                'RMSE', 'L2': Root Mean of Squared Errors / L2 Norm
                default: 'M' """

        norm = norm.upper()

        # Mean of Values (M)
        if norm in ["M"]:
            return numpy.mean(data, axis=0)
        # Mean of Errors (ME)
        if norm in ["ME"]:
            return numpy.mean(numpy.abs(data), axis=0)
        # Mean of Squared Errors (MSE)
        if norm in ["MSE"]:
            return numpy.mean(data ** 2, axis=0)
        # Root Mean of Squared Errors (RMSE) / L2-Norm (L2)
        if norm in ["RMSE", "L2"]:
            return numpy.sqrt(numpy.mean(data ** 2, axis=0))

        return nemoa.log("error", "unknown data mean norm '%s'" % (norm))
示例#53
0
文件: base.py 项目: Jackieee/nemoa
    def getDataSum(data, norm="S"):
        """Return sum of data.

        Keyword Arguments:
            data -- numpy array containing data
            norm -- data mean norm
                'S': Sum of Values
                'SE', 'L1': Sum of Errors / L1 Norm
                'SSE': Sum of Squared Errors
                'RSSE': Root Sum of Squared Errors
                default: 'S' """

        norm = norm.upper()

        # Sum of Values (S)
        if norm in ["S"]:
            return numpy.sum(data, axis=0)
        # Sum of Errors (SE) / L1-Norm (L1)
        if norm in ["SE", "L1"]:
            return numpy.sum(numpy.abs(data), axis=0)
        # Sum of Squared Errors (SSE)
        if norm in ["SSE"]:
            return numpy.sum(data ** 2, axis=0)
        # Root Sum of Squared Errors (RSSE)
        if norm in ["RSSE"]:
            return numpy.sqrt(numpy.sum(data ** 2, axis=0))

        return nemoa.log("error", "unknown data sum norm '%s'" % (norm))
示例#54
0
文件: config.py 项目: Jackieee/nemoa
    def getPath(self, str, check = False, create = False):
        """Resolve path."""

        # clean up input string
        path = str.strip()

        # expand unix home directory
        path = os.path.expanduser(path)

        # expand nemoa environment variables
        path = self.__expandPath(path)

        # expand unix environment variables
        path = os.path.expandvars(path)

        # create directory
        if create:
            dir = os.path.dirname(path)
            if not os.path.exists(dir): os.makedirs(dir)

        # check path
        if check and not os.path.exists(path): return nemoa.log('warning',
            "directory '%s' does not exist!" % (path))

        return path
示例#55
0
def getConfig(type = None, config = None, merge = ['params'], **kwargs):
    """Return object configuration as dictionary."""
    if config == None: return {}
    if isinstance(config, dict): return copy.deepcopy(config)
    elif not isinstance(config, str) or not isinstance(type, str): return False

    name, params = nemoa.common.strSplitParams(config)
    if 'params' in kwargs and isinstance(kwargs['params'], dict):
        params = nemoa.common.dictMerge(kwargs['params'], params)

    search = [name, '%s.%s' % (project(), name),
        name + '.default', 'base.' + name]
    if isinstance(config, str):
        search += [config, '%s.%s' % (project(), config),
        config + '.default', 'base.' + config]

    # get config
    if not name: name = project() + '.default'
    for cfgName in search:
        cfg = __shared['config'].get(
            type = type, name = cfgName, merge = merge, params = params)
        if isinstance(cfg, dict): return cfg

    return nemoa.log('error', """
        could not get configuration:
        no %s with name '%s' could be found""" % (type, name))
示例#56
0
文件: base.py 项目: Jackieee/nemoa
    def getCorruptedData(self, data, type = None, factor = 0.5):
        """Return numpy array with (partly) corrupted data.

        Keyword Arguments:
            type -- string describing algorithm for corruption
                'mask': Masking Noise
                    A fraction of every sample is forced to zero
                'gauss': Gaussian Noise
                    Additive isotropic Gaussian noise
                'salt': Salt-and-pepper noise
                    A fraction of every sample is forced to min or max
                    with equal possibility
            factor -- float describing the strengt of the corruption
                The influence of the parameter depends on the
                type of the corruption"""

        if type in [None, 'none']: return data
        elif type == 'mask': return data * numpy.random.binomial(
            size = data.shape, n = 1, p = 1 - factor)
        elif type == 'gauss': return data + numpy.random.normal(
            size = data.shape, loc = 0.0, scale = factor)
        #2do!: implement salt and pepper noise
        #elif type == 'salt': return
        else: return nemoa.log('error',
            "unkown data corruption type '%s'!" % (type))