示例#1
0
def gather_mpi_arguments(hostfile, params):
    print_and_log(['MPI detected: %s' % str(MPI_VENDOR)], 'debug', logger)
    if MPI_VENDOR[0] == 'Open MPI':
        mpi_args = ['mpirun']
        if os.getenv('LD_LIBRARY_PATH'):
            mpi_args += ['-x', 'LD_LIBRARY_PATH']
        if os.getenv('PATH'):
            mpi_args += ['-x', 'PATH']
        if os.getenv('PYTHONPATH'):
            mpi_args += ['-x', 'PYTHONPATH']
        if os.path.exists(hostfile):
            mpi_args += ['-hostfile', hostfile]
    elif MPI_VENDOR[0] == 'Microsoft MPI':
        mpi_args = ['mpiexec']
        if os.path.exists(hostfile):
            mpi_args += ['-machinefile', hostfile]
    elif MPI_VENDOR[0] == 'MPICH2':
        mpi_args = ['mpiexec']
        if os.path.exists(hostfile):
            mpi_args += ['-f', hostfile]
    elif MPI_VENDOR[0] == 'MPICH':
        mpi_args = ['mpiexec']
        if os.path.exists(hostfile):
            mpi_args += ['-f', hostfile]
    else:
        print_and_log([
            '%s may not be yet properly implemented: contact developpers' %
            MPI_VENDOR[0]
        ], 'error', logger)
        mpi_args = ['mpirun']
        if os.path.exists(hostfile):
            mpi_args += ['-hostfile', hostfile]
    return mpi_args
示例#2
0
    def getboolean(self, section, data, check=True):
        """
        Gets the boolean variable from the variable data in a section

        Parameters
        ----------
        section :  str
            the section in *params to be read (e.g., 'filtering')
        
        data : str
            the variable data to be read (e.g., 'remove_median')

        Returns
        -------
        bool
            if the variable data is applied or not.    
        
        """
        if check is False:
            return self.parser.getboolean(section, data)

        try:
            return self.parser.getboolean(section, data)
        except Exception:
            if comm.rank == 0:
                print_and_log([
                    "Parameter %s is missing in section [%s]" % (data, section)
                ], 'error', logger)
            sys.exit(0)
示例#3
0
    def getint(self, section, data, check=True):
        """
        Gets the value of the  variable data in a section

        Parameters
        ----------
        section :  str
            the section in *params to be read (e.g., 'detection')
        
        data : str
            the variable data to be read (e.g., 'oversampling_factor')

        Returns
        -------
        int 
            The value of the variable data.    
        
        """
        if check is False:
            return self.parser.getint(section, data)

        try:
            return self.parser.getint(section, data)
        except Exception:
            if comm.rank == 0:
                print_and_log([
                    "Parameter %s is missing in section [%s]" % (data, section)
                ], 'error', logger)
            sys.exit(0)
示例#4
0
    def _create_data_file(self, data_file, is_empty, params, stream_mode):
        file_format = params.pop('file_format').lower()
        if comm.rank == 0:
            print_and_log(
                ['Trying to read file %s as %s' % (data_file, file_format)],
                'debug', logger)

        data = __supported_data_files__[file_format](data_file, params,
                                                     is_empty, stream_mode)
        self.rate = data.sampling_rate
        self.nb_channels = data.nb_channels
        self.gain = data.gain
        self.data_file = data
        self._update_rate_values()

        N_e = self.getint('data', 'N_e')
        if N_e > self.nb_channels:
            if comm.rank == 0:
                print_and_log([
                    'Analyzed %d channels but only %d are recorded' %
                    (N_e, self.nb_channels)
                ], 'error', logger)
            sys.exit(0)

        return data
示例#5
0
def purge(file, pattern):
    dir = os.path.dirname(os.path.abspath(file))
    for f in os.listdir(dir):
        if f.find(pattern) > -1:
            os.remove(os.path.join(dir, f))
    if comm.rank == 0:
        print_and_log(['Removing %s for directory %s' %(pattern, dir)], 'debug', logger)
示例#6
0
def test_mpi_ring(nb_nodes):
    if comm.size != nb_nodes:
        if comm.rank == 0:
            print_and_log([
                'The MPI install does not seems to be correct!',
                'Be sure mpi4py has been compiled for your default version!'
            ], 'warning', logger)
        sys.exit()
示例#7
0
    def _update_rate_values(self):

        if self._N_t is None:

            if comm.rank == 0:
                print_and_log(
                    ['Changing all values in the param depending on the rate'],
                    'debug', logger)

            try:
                self._N_t = self.getfloat('detection', 'N_t')
            except Exception:
                if comm.rank == 0:
                    print_and_log(
                        ['N_t must now be defined in the [detection] section'],
                        'error', logger)
                sys.exit(0)

            self._N_t = int(self.rate * self._N_t * 1e-3)

            jitter_range = self.getfloat('detection', 'jitter_range')
            self.set('detection', 'jitter_range',
                     str(int(self.rate * jitter_range * 1e-3)))
            if numpy.mod(self._N_t, 2) == 0:
                self._N_t += 1

            self.set('detection', 'N_t', str(self._N_t))
            self.set('detection', 'dist_peaks', str(self._N_t))
            self.set('detection', 'template_shift', str((self._N_t - 1) // 2))

            self._savgol = int(self.rate * 0.5 * 1e-3)
            if numpy.mod(self._savgol, 2) == 0:
                self._savgol += 1

            self.set('clustering', 'savgol_window', str(self._savgol))

            if self.parser._sections['fitting'].has_key('chunk'):
                self.parser.set('fitting', 'chunk_size',
                                self.parser._sections['fitting']['chunk'])

            for section in ['data', 'whitening', 'fitting']:
                chunk_size = int(
                    self.parser.getfloat(section, 'chunk_size') * self.rate)
                self.set(section, 'chunk_size', str(chunk_size))

            for section in ['clustering', 'whitening', 'extracting']:
                safety_time = self.get(section, 'safety_time')
                if safety_time == 'auto':
                    self.set(section, 'safety_time', str(self._N_t // 3))
                else:
                    safety_time = float(safety_time)
                    self.set(section, 'safety_time',
                             str(int(safety_time * self.rate * 1e-3)))

            refractory = self.getfloat('fitting', 'refractory')
            self.set('fitting', 'refractory',
                     str(int(refractory * self.rate * 1e-3)))
示例#8
0
    def write(self, section, flag, value, preview_path=False):
        if comm.rank == 0:
            print_and_log(
                ['Writing value %s for %s:%s' % (value, section, flag)],
                'debug', logger)
        self.parser.set(section, flag, value)
        if preview_path:
            f = open(self.get('data', 'preview_path'), 'r')
        else:
            f = open(self.file_params, 'r')

        lines = f.readlines()
        f.close()
        spaces = ''.join([' '] * (max(0, 15 - len(flag))))

        to_write = '%s%s= %s              #!! AUTOMATICALLY EDITED: DO NOT MODIFY !!\n' % (
            flag, spaces, value)

        section_area = [0, len(lines)]
        idx = 0
        for count, line in enumerate(lines):

            if (idx == 1) and line.strip().replace('[', '').replace(
                    ']', '') in self.__all_sections__:
                section_area[idx] = count
                idx += 1

            if (line.find('[%s]' % section) > -1):
                section_area[idx] = count
                idx += 1

        has_been_changed = False

        for count in xrange(section_area[0] + 1, section_area[1]):
            if '=' in lines[count]:
                key = lines[count].split('=')[0].replace(' ', '')
                if key == flag:
                    lines[count] = to_write
                    has_been_changed = True

        if not has_been_changed:
            lines.insert(section_area[1] - 1, to_write)

        if preview_path:
            f = open(self.get('data', 'preview_path'), 'w')
        else:
            f = open(self.file_params, 'w')
        for line in lines:
            f.write(line)
        f.close()
示例#9
0
def read_probe(parser):
    probe = {}
    filename = os.path.abspath(
        os.path.expanduser(parser.get('data', 'mapping')))
    if comm.rank == 0:
        print_and_log(["Reading the probe file %s" % filename], 'debug',
                      logger)
    if not os.path.exists(filename):
        if comm.rank == 0:
            print_and_log(["The probe file %s can not be found" % filename],
                          'error', logger)
        sys.exit(1)
    try:
        with open(filename, 'r') as f:
            probetext = f.read()
            exec(probetext, probe)
    except Exception as ex:
        if comm.rank == 0:
            print_and_log([
                "Something wrong with the syntax of the probe file:\n" +
                str(ex)
            ], 'error', logger)
        sys.exit(1)

    key_flags = ['total_nb_channels', 'radius', 'channel_groups']
    for key in key_flags:
        if not probe.has_key(key):
            if comm.rank == 0:
                print_and_log(["%s is missing in the probe file" % key],
                              'error', logger)
            sys.exit(1)
    return probe
示例#10
0
def parse_dead_channels(channels):
    is_correct = False
    try:
        dead_channels = ast.literal_eval(channels)
        is_correct = True
    except Exception:
        pass

    if not is_correct:
        if comm.rank == 0:
            print_and_log(["The syntax for dead channels is not correct!"],
                          'error', logger)
        sys.exit(0)
    else:
        return dead_channels
示例#11
0
def test_patch_for_similarities(params, extension):

    file_out_suff = params.get('data', 'file_out_suff')
    template_file = file_out_suff + '.templates%s.hdf5' % extension
    if os.path.exists(template_file):
        version = io.load_data(params, 'version', extension)
    else:
        print_and_log(['No templates found! Check suffix?'], 'error', logger)
        sys.exit(0)

    if version is not None:
        if (StrictVersion(version) >= StrictVersion('0.6.0')):
            return True
    else:
        print_and_log(["Version is below 0.6.0"], 'debug', logger)
        return False
示例#12
0
    def get_data_file(self, is_empty=False, params=None, source=False, has_been_created=True):

        if params is None:
            params = {}

        for key, value in self.parser._sections['data'].items():
            if key not in params:
                params[key] = value

        data_file     = params.pop('data_file')
        stream_mode   = self.get('data', 'stream_mode').lower()

        if stream_mode in ['none']:
            stream_mode = None

        if not self.getboolean('data', 'overwrite'):
            # If we do not want to overwrite, we first read the original data file
            # Then, if we do not want to obtain it as a source file, we switch the
            # format to raw_binary and the output file name

            if not source:

                # First we read the original data file, that should not be empty
                print_and_log(['Reading first the real data file to get the parameters'], 'debug', logger)
                tmp = self._create_data_file(data_file, False, params, stream_mode)

                # Then we change the dataa_file name
                data_file = self.get('data', 'data_file_no_overwrite')

                if comm.rank == 0:
                    print_and_log(['Forcing the exported data file to be of type raw_binary'], 'debug', logger)

                # And we force the results to be of type float32, without streams
                params['file_format']   = 'raw_binary'
                params['data_dtype']    = 'float32'
                params['dtype_offset']  = 0
                params['data_offset']   = 0
                params['sampling_rate'] = self.rate
                params['nb_channels']   = self.nb_channels
                params['gain']          = self.gain
                stream_mode             = None
                data_file, extension    = os.path.splitext(data_file)
                data_file              += ".dat"

            else:
                if has_been_created:
                  data_file = self.get('data', 'data_file_no_overwrite')
                  if not os.path.exists(data_file):
                      if comm.rank== 0:
                          print_and_log(['The overwrite option is only valid if the filtering step is launched before!'], 'error', logger)
                      sys.exit(1)
                else:
                    if comm.rank== 0:
                        print_and_log(['The copy file has not yet been created! Returns normal file'], 'debug', logger)

        return self._create_data_file(data_file, is_empty, params, stream_mode)
示例#13
0
    def _create_data_file(self, data_file, is_empty, params, stream_mode):
        """
        Creates a data file with parameters from the param file.

        Parameters
        ----------

        data_file : str 
            the path for the datafile
        
        is_empty : bool

        params :  dict

        stream_mode : str
            multi-files, multi-folders or single-file.

        Returns
        -------
        dict
            a supported data file with the parameters from the param file.
        """
        file_format = params.pop('file_format').lower()
        if comm.rank == 0:
            print_and_log(
                ['Trying to read file %s as %s' % (data_file, file_format)],
                'debug', logger)

        data = __supported_data_files__[file_format](data_file, params,
                                                     is_empty, stream_mode)
        self.rate = data.sampling_rate
        self.nb_channels = data.nb_channels
        self.gain = data.gain
        self.data_file = data
        self._update_rate_values()

        N_e = self.getint('data', 'N_e')
        if N_e > self.nb_channels:
            if comm.rank == 0:
                print_and_log([
                    'Analyzed %d channels but only %d are recorded' %
                    (N_e, self.nb_channels)
                ], 'error', logger)
            sys.exit(0)

        return data
示例#14
0
def detect_memory(params, whitening=False, filtering=False, fitting=False):
    from psutil import virtual_memory

    N_e = params.getint('data', 'N_e')
    safety_threshold = params.getfloat('data', 'memory_usage')
    data_file = params.data_file
    data_file.open()
    sampling_rate = data_file.sampling_rate
    duation = data_file.duration
    data_file.close()

    from uuid import getnode as get_mac
    myip = numpy.int64(get_mac()) % 100000
    sub_comm = comm.Split_type(MPI.COMM_TYPE_SHARED, myip)

    res = numpy.zeros(1, dtype=numpy.int64)
    mem = virtual_memory()

    if sub_comm.rank == 0:
        res[0] = safety_threshold * numpy.int64(mem.available // sub_comm.size)

    sub_comm.Barrier()
    sub_comm.Free()

    memory = all_gather_array(res, comm, 1, 'int64')

    idx = numpy.where(memory > 0)
    max_memory = numpy.min(memory[idx]) // (4 * N_e)

    if whitening or filtering:
        max_size = int(30 * data_file.sampling_rate)
    elif fitting:
        max_size = int(0.1 * data_file.sampling_rate)
    else:
        max_size = (data_file.duration // comm.size)

    chunk_size = min(max_memory, max_size)

    if comm.rank == 0:
        print_and_log([
            'Setting data chunk size to %g second' %
            (chunk_size / float(sampling_rate))
        ], 'debug', logger)

    return chunk_size
示例#15
0
def read_probe(parser, radius_in_probe=True):
    """
    Read the probe file 

    Arguments
    ---------
    parser :
        The circur parser object.

    radius_in_probe: bool
        True (default) if radius is read from the probe.
        False if is read from params file

    Returns
    -------
    The probe
    """
    probe = {}
    filename = os.path.abspath(
        os.path.expanduser(parser.get('data', 'mapping')))
    if comm.rank == 0:
        print_and_log(["Reading the probe file %s" % filename], 'debug',
                      logger)
    if not os.path.exists(filename):
        if comm.rank == 0:
            print_and_log(["The probe file %s can not be found" % filename],
                          'error', logger)
        sys.exit(0)
    try:
        with open(filename, 'r') as f:
            probetext = f.read()
            exec(probetext, probe)
            del probe['__builtins__']
    except Exception as ex:
        if comm.rank == 0:
            print_and_log([
                "Something wrong with the syntax of the probe file:\n" +
                str(ex)
            ], 'error', logger)
        sys.exit(0)

    key_flags = ['total_nb_channels', 'channel_groups']
    if radius_in_probe:
        key_flags += ['radius']

    for key in key_flags:
        if not probe.has_key(key):
            if comm.rank == 0:
                print_and_log(["%s is missing in the probe file" % key],
                              'error', logger)
            sys.exit(0)
    return probe
示例#16
0
def test_patch_for_similarities(params, extension):

    file_out_suff = params.get('data', 'file_out_suff')
    template_file = file_out_suff + '.templates%s.hdf5' % extension
    if os.path.exists(template_file):
        try:
            version = h5py.File(
                template_file, 'r',
                libver='latest').get('version')[0].decode('ascii')
        except Exception:
            version = None
    else:
        raise Exception('No templates found! Check suffix?')

    if version is not None:
        if (StrictVersion(version) >= StrictVersion('0.6.0')):
            return True
    else:
        print_and_log(["Version is below 0.6.0"], 'debug', logger)
        return False
示例#17
0
def detect_memory(params, safety_threshold=0.1):
    from psutil import virtual_memory

    N_e = params.getint('data', 'N_e')
    data_file = params.data_file
    data_file.open()
    sampling_rate = data_file.sampling_rate
    duation = data_file.duration
    data_file.close()

    from uuid import getnode as get_mac
    myip = numpy.int64(get_mac()) % 100000
    sub_comm = comm.Split_type(MPI.COMM_TYPE_SHARED, myip)

    res = numpy.zeros(1, dtype=numpy.int64)
    mem = virtual_memory()

    if sub_comm.rank == 0:
        res[0] = safety_threshold * numpy.int64(mem.available // sub_comm.size)

    sub_comm.Barrier()
    sub_comm.Free()

    memory = all_gather_array(res, comm, 1, 'int64')

    idx = numpy.where(memory > 0)
    max_memory = numpy.min(memory[idx]) // (4 * N_e)
    max_size = (data_file.duration // comm.size)

    for section in ['data', 'whitening']:
        chunk_size = min(max_memory, max_size)
        params.set(section, 'chunk_size', str(chunk_size))

    if comm.rank == 0:
        print_and_log([
            'Setting data chunk size to %d second' %
            (max_size / float(sampling_rate))
        ], 'info', logger)

    return params
示例#18
0
def gather_mpi_arguments(hostfile, params):
    print_and_log(['MPI detected: %s' % str(MPI_VENDOR)], 'debug', logger)
    if MPI_VENDOR[0] == 'Open MPI':
        if MPI_VENDOR[1][0] >= 3:
            print_and_log([
                'SpyKING CIRCUS does not work with OPENMPI >= 3.0',
                'Consider downgrading or switching to MPICH'
            ], 'error', logger)
            sys.exit(0)
        mpi_args = ['mpirun']
        if os.getenv('LD_LIBRARY_PATH'):
            mpi_args += ['-x', 'LD_LIBRARY_PATH']
        if os.getenv('PATH'):
            mpi_args += ['-x', 'PATH']
        if os.getenv('PYTHONPATH'):
            mpi_args += ['-x', 'PYTHONPATH']
        if os.path.exists(hostfile):
            mpi_args += ['-hostfile', hostfile]
    elif MPI_VENDOR[0] == 'Microsoft MPI':
        mpi_args = ['mpiexec']
        if os.path.exists(hostfile):
            mpi_args += ['-machinefile', hostfile]
    elif MPI_VENDOR[0] == 'MPICH2':
        mpi_args = ['mpiexec']
        if os.path.exists(hostfile):
            mpi_args += ['-f', hostfile]
    elif MPI_VENDOR[0] == 'MPICH':
        mpi_args = ['mpiexec']
        if os.path.exists(hostfile):
            mpi_args += ['-f', hostfile]
    else:
        print_and_log([
            '%s may not be yet properly implemented: contact developpers' %
            MPI_VENDOR[0]
        ], 'error', logger)
        mpi_args = ['mpirun']
        if os.path.exists(hostfile):
            mpi_args += ['-hostfile', hostfile]
    return mpi_args
示例#19
0
def apply_patch_for_similarities(params, extension):

    if not test_patch_for_similarities(params, extension):

        file_out_suff = params.get('data', 'file_out_suff')
        hdf5_compress = params.getboolean('data', 'hdf5_compress')
        blosc_compress = params.getboolean('data', 'blosc_compress')
        N_tm = io.load_data(params, 'nb_templates', extension)
        N_half = int(N_tm // 2)
        N_t = params.getint('detection', 'N_t')
        duration = 2 * N_t - 1

        if comm.rank == 0:
            print_and_log(["Fixing overlaps from 0.5.XX..."], 'default',
                          logger)

        maxlag = numpy.zeros((N_half, N_half), dtype=numpy.int32)
        maxoverlap = numpy.zeros((N_half, N_half), dtype=numpy.float32)

        to_explore = numpy.arange(N_half - 1)[comm.rank::comm.size]

        if comm.rank == 0:
            to_explore = get_tqdm_progressbar(to_explore)

        if not SHARED_MEMORY:
            over_x, over_y, over_data, over_shape = io.load_data(
                params, 'overlaps-raw', extension=extension)
        else:
            over_x, over_y, over_data, over_shape = io.load_data_memshared(
                params, 'overlaps-raw', extension=extension)

        for i in to_explore:

            idx = numpy.where((over_x >= i * N_tm + i + 1)
                              & (over_x < (i * N_tm + N_half)))[0]
            local_x = over_x[idx] - (i * N_tm + i + 1)
            data = numpy.zeros((N_half - (i + 1), duration),
                               dtype=numpy.float32)
            data[local_x, over_y[idx]] = over_data[idx]
            maxlag[i, i + 1:] = N_t - numpy.argmax(data, 1)
            maxlag[i + 1:, i] = -maxlag[i, i + 1:]
            maxoverlap[i, i + 1:] = numpy.max(data, 1)
            maxoverlap[i + 1:, i] = maxoverlap[i, i + 1:]

        #Now we need to sync everything across nodes
        maxlag = gather_array(maxlag,
                              comm,
                              0,
                              1,
                              'int32',
                              compress=blosc_compress)

        if comm.rank == 0:
            maxlag = maxlag.reshape(comm.size, N_half, N_half)
            maxlag = numpy.sum(maxlag, 0)

        maxoverlap = gather_array(maxoverlap,
                                  comm,
                                  0,
                                  1,
                                  'float32',
                                  compress=blosc_compress)
        if comm.rank == 0:
            maxoverlap = maxoverlap.reshape(comm.size, N_half, N_half)
            maxoverlap = numpy.sum(maxoverlap, 0)

        if comm.rank == 0:
            myfile2 = h5py.File(file_out_suff +
                                '.templates%s.hdf5' % extension,
                                'r+',
                                libver='earliest')

            for key in ['maxoverlap', 'maxlag', 'version']:
                if key in myfile2.keys():
                    myfile2.pop(key)

            myfile2.create_dataset('version',
                                   data=numpy.array(
                                       circus.__version__.split('.'),
                                       dtype=numpy.int32))
            if hdf5_compress:
                myfile2.create_dataset('maxlag',
                                       data=maxlag,
                                       compression='gzip')
                myfile2.create_dataset('maxoverlap',
                                       data=maxoverlap,
                                       compression='gzip')
            else:
                myfile2.create_dataset('maxlag', data=maxlag)
                myfile2.create_dataset('maxoverlap', data=maxoverlap)
            myfile2.close()
示例#20
0
    def __init__(self, file_name, **kwargs):

        self.file_name    = os.path.abspath(file_name)
        f_next, extension = os.path.splitext(self.file_name)
        file_path         = os.path.dirname(self.file_name)
        self.file_params  = f_next + '.params'
        self.logfile      = f_next + '.log'
        self.parser       = configparser.ConfigParser()
        self._N_t         = None

        if not os.path.exists(self.file_params):
            if comm.rank == 0:
                print_and_log(["%s does not exist" %self.file_params], 'error', logger)
            sys.exit(1)

        if comm.rank == 0:
            print_and_log(['Creating a Circus Parser for datafile %s' %self.file_name], 'debug', logger)
        self.parser.read(self.file_params)

        for section in self.__all_sections__:
            if self.parser.has_section(section):
                for (key, value) in self.parser.items(section):
                    self.parser.set(section, key, value.split('#')[0].rstrip())
            else:
                self.parser.add_section(section)

        for item in self.__default_values__ + self.__extra_values__:
            section, name, val_type, value = item
            try:
                if val_type is 'bool':
                    self.parser.getboolean(section, name)
                elif val_type is 'int':
                    self.parser.getint(section, name)
                elif val_type is 'float':
                    self.parser.getfloat(section, name)
                elif val_type is 'string':
                    self.parser.get(section, name)
            except Exception:
                self.parser.set(section, name, value)

        for key, value in kwargs.items():
            for section in self.__all_sections__:
                if self.parser._sections[section].has_key(key):
                    self.parser._sections[section][key] = value


        self.probe = read_probe(self.parser)

        N_e = 0
        for key in self.probe['channel_groups'].keys():
            N_e += len(self.probe['channel_groups'][key]['channels'])

        self.set('data', 'N_e', str(N_e))
        self.set('data', 'N_total', str(self.probe['total_nb_channels']))
        self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
        self.nb_channels = self.probe['total_nb_channels']

        if N_e > self.nb_channels:
            if comm.rank == 0:
                print_and_log(['The number of analyzed channels is higher than the number of recorded channels'], 'error', logger)
            sys.exit(1)

        to_write = ["You must specify explicitly the file format in the config file",
                    "Please have a look to the documentation and add a file_format",
                    "parameter in the [data] section. Valid files formats can be:", '']
        try:
            self.file_format = self.parser.get('data', 'file_format')
        except Exception:
            if comm.rank == 0:
                for f in __supported_data_files__.keys():
                    to_write += ['-- %s -- %s' %(f, __supported_data_files__[f].extension)]

                to_write += ['', "To get more info on a given file format, see",
                    ">> spyking-circus file_format -i"]

                print_and_log(to_write, 'error', logger)
            sys.exit(1)

        test = self.file_format.lower() in __supported_data_files__.keys()
        if not test:
            if comm.rank == 0:
                for f in __supported_data_files__.keys():
                    to_write += ['-- %s -- %s' %(f, __supported_data_files__[f].extension)]

                to_write += ['', "To get more info on a given file format, see",
                    ">> spyking-circus file_format -i"]

                print_and_log(to_write, 'error', logger)
            sys.exit(1)


        try:
            self.parser.get('detection', 'radius')
        except Exception:
            self.parser.set('detection', 'radius', 'auto')
        try:
            self.parser.getint('detection', 'radius')
        except Exception:
            self.parser.set('detection', 'radius', str(int(self.probe['radius'])))

        if self.parser.getboolean('triggers', 'clean_artefact'):
            if (self.parser.get('triggers', 'trig_file') == '') or (self.parser.get('triggers', 'trig_windows') == ''):
                if comm.rank == 0:
                    print_and_log(["trig_file and trig_windows must be specified in [triggers]"], 'error', logger)
                sys.exit(1)

        units = ['ms', 'timestep']
        test = self.parser.get('triggers', 'trig_unit').lower() in units
        if not test:
          if comm.rank == 0:
              print_and_log(["trig_unit in [triggers] should be in %s" %str(units)], 'error', logger)
          sys.exit(1)
        else:
          self.parser.set('triggers', 'trig_in_ms', str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))

        self.parser.set('triggers', 'trig_file', os.path.abspath(os.path.expanduser(self.parser.get('triggers', 'trig_file'))))
        self.parser.set('triggers', 'trig_windows', os.path.abspath(os.path.expanduser(self.parser.get('triggers', 'trig_windows'))))

        units = ['ms', 'timestep']
        test = self.parser.get('triggers', 'dead_unit').lower() in units
        if not test:
          if comm.rank == 0:
              print_and_log(["dead_unit in [triggers] should be in %s" %str(units)], 'error', logger)
          sys.exit(1)
        else:
          self.parser.set('triggers', 'dead_in_ms', str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))

        self.parser.set('triggers', 'dead_file', os.path.abspath(os.path.expanduser(self.parser.get('triggers', 'dead_file'))))

        test = (self.parser.get('clustering', 'extraction').lower() in ['median-raw', 'median-pca', 'mean-raw', 'mean-pca'])
        if not test:
            if comm.rank == 0:
                print_and_log(["Only 4 extraction modes in [clustering]: median-raw, median-pca, mean-raw or mean-pca!"], 'error', logger)
            sys.exit(1)

        test = (self.parser.get('detection', 'peaks').lower() in ['negative', 'positive', 'both'])
        if not test:
            if comm.rank == 0:
                print_and_log(["Only 3 detection modes for peaks in [detection]: negative, positive, both"], 'error', logger)

        try:
            os.makedirs(f_next)
        except Exception:
            pass

        self.parser.set('data', 'data_file', self.file_name)
        self.parser.set('data', 'data_file_no_overwrite', os.path.join(file_path, os.path.basename(f_next) + '_all_sc.dat'))
        file_out = os.path.join(f_next, os.path.basename(f_next))
        self.parser.set('data', 'file_out', file_out) # Output file without suffix
        self.parser.set('data', 'file_out_suff', file_out  + self.parser.get('data', 'suffix')) # Output file with suffix
        self.parser.set('data', 'data_file_noext', f_next)   # Data file (assuming .filtered at the end)

        for section in ['whitening', 'clustering']:
            test = (self.parser.getfloat(section, 'nb_elts') > 0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
            if not test:
                if comm.rank == 0:
                    print_and_log(["nb_elts in [%s] should be in [0,1]" %section], 'error', logger)
                sys.exit(1)

        test = (self.parser.getfloat('clustering', 'nclus_min') >= 0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["nclus_min in [clustering] should be in [0,1["], 'error', logger)
            sys.exit(1)

        test = (self.parser.getfloat('clustering', 'noise_thr') >= 0) and (self.parser.getfloat('clustering', 'noise_thr') <= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["noise_thr in [clustering] should be in [0,1]"], 'error', logger)
            sys.exit(1)

        test = (self.parser.getfloat('validating', 'test_size') > 0) and (self.parser.getfloat('validating', 'test_size') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["test_size in [validating] should be in ]0,1["], 'error', logger)
            sys.exit(1)

        test = (self.parser.getfloat('clustering', 'cc_merge') >= 0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["cc_merge in [validating] should be in [0,1]"], 'error', logger)
            sys.exit(1)

        test = (self.parser.getfloat('clustering', 'nclus_min') >= 0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["nclus_min in [validating] should be in [0,1["], 'error', logger)
            sys.exit(1)

        test = (self.parser.getfloat('merging', 'auto_mode') >= 0) and (self.parser.getfloat('merging', 'auto_mode') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["auto_mode in [merging] should be in [0,1["], 'error', logger)
            sys.exit(1)

        fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
        for section in ['clustering', 'validating', 'triggers']:
          test = self.parser.get('clustering', 'make_plots').lower() in fileformats
          if not test:
              if comm.rank == 0:
                  print_and_log(["make_plots in [%s] should be in %s" %(section, str(fileformats))], 'error', logger)
              sys.exit(1)

        dispersion     = self.parser.get('clustering', 'dispersion').replace('(', '').replace(')', '').split(',')
        dispersion     = map(float, dispersion)
        test =  (0 < dispersion[0]) and (0 < dispersion[1])
        if not test:
            if comm.rank == 0:
                print_and_log(["min and max dispersions in [clustering] should be positive"], 'error', logger)
            sys.exit(1)


        pcs_export = ['prompt', 'none', 'all', 'some']
        test = self.parser.get('converting', 'export_pcs').lower() in pcs_export
        if not test:
            if comm.rank == 0:
                print_and_log(["export_pcs in [converting] should be in %s" %str(pcs_export)], 'error', logger)
            sys.exit(1)
        else:
            if self.parser.get('converting', 'export_pcs').lower() == 'none':
                self.parser.set('converting', 'export_pcs', 'n')
            elif self.parser.get('converting', 'export_pcs').lower() == 'some':
                self.parser.set('converting', 'export_pcs', 's')
            elif self.parser.get('converting', 'export_pcs').lower() == 'all':
                self.parser.set('converting', 'export_pcs', 'a')
示例#21
0
    def __init__(self, file_name, **kwargs):

        self.file_name = os.path.abspath(file_name)
        f_next, extension = os.path.splitext(self.file_name)
        file_path = os.path.dirname(self.file_name)
        self.file_params = f_next + '.params'
        self.logfile = f_next + '.log'
        self.parser = configparser.ConfigParser()

        ## First, we remove all tabulations from the parameter file, in order
        ## to secure the parser
        if comm.rank == 0:
            myfile = open(self.file_params, 'r')
            lines = myfile.readlines()
            myfile.close()
            myfile = open(self.file_params, 'w')
            for l in lines:
                myfile.write(l.replace('\t', ''))
            myfile.close()

        comm.Barrier()

        self._N_t = None

        if not os.path.exists(self.file_params):
            if comm.rank == 0:
                print_and_log(["%s does not exist" % self.file_params],
                              'error', logger)
            sys.exit(0)

        if comm.rank == 0:
            print_and_log(
                ['Creating a Circus Parser for datafile %s' % self.file_name],
                'debug', logger)
        self.parser.read(self.file_params)

        for section in self.__all_sections__:
            if self.parser.has_section(section):
                for (key, value) in self.parser.items(section):
                    self.parser.set(section, key, value.split('#')[0].rstrip())
            else:
                self.parser.add_section(section)

        for item in self.__default_values__ + self.__extra_values__:
            section, name, val_type, value = item
            try:
                if val_type is 'bool':
                    self.parser.getboolean(section, name)
                elif val_type is 'int':
                    self.parser.getint(section, name)
                elif val_type is 'float':
                    self.parser.getfloat(section, name)
                elif val_type is 'string':
                    self.parser.get(section, name)
            except Exception:
                self.parser.set(section, name, value)

        for key, value in list(kwargs.items()):
            for section in self.__all_sections__:
                if key in self.parser._sections[section]:
                    self.parser._sections[section][key] = value

        self.probe = read_probe(self.parser)

        dead_channels = self.parser.get('detection', 'dead_channels')
        if dead_channels != '':
            dead_channels = parse_dead_channels(dead_channels)
            if comm.rank == 0:
                print_and_log(
                    ["Removing dead channels %s" % str(dead_channels)],
                    'debug', logger)
            for key in dead_channels.keys():
                if key in self.probe["channel_groups"].keys():
                    for channel in dead_channels[key]:
                        n_before = len(
                            self.probe["channel_groups"][key]['channels'])
                        self.probe["channel_groups"][key]['channels'] = list(
                            set(self.probe["channel_groups"][key]
                                ['channels']).difference(dead_channels[key]))
                        n_after = len(
                            self.probe["channel_groups"][key]['channels'])
                else:
                    if comm.rank == 0:
                        print_and_log([
                            "Probe has no group named %s for dead channels" %
                            key
                        ], 'debug', logger)

        N_e = 0
        for key in list(self.probe['channel_groups'].keys()):
            N_e += len(self.probe['channel_groups'][key]['channels'])

        self.set('data', 'N_e', str(N_e))
        self.set('data', 'N_total', str(self.probe['total_nb_channels']))
        self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
        self.nb_channels = self.probe['total_nb_channels']

        if N_e > self.nb_channels:
            if comm.rank == 0:
                print_and_log([
                    'The number of analyzed channels is higher than the number of recorded channels'
                ], 'error', logger)
            sys.exit(0)

        if N_e == 1 and self.parser.getboolean('filtering', 'remove_median'):
            if comm.rank == 0:
                print_and_log([
                    "With 1 channel, remove_median in [filtering] is not possible"
                ], 'error', logger)
            sys.exit(0)

        to_write = [
            "You must specify explicitly the file format in the config file",
            "Please have a look to the documentation and add a file_format",
            "parameter in the [data] section. Valid files formats can be:", ''
        ]
        try:
            self.file_format = self.parser.get('data', 'file_format')
        except Exception:
            if comm.rank == 0:
                for f in list(__supported_data_files__.keys()):
                    to_write += [
                        '-- %s -- %s' %
                        (f, __supported_data_files__[f].extension)
                    ]

                to_write += [
                    '', "To get more info on a given file format, see",
                    ">> spyking-circus file_format -i"
                ]

                print_and_log(to_write, 'error', logger)
            sys.exit(0)

        test = self.file_format.lower() in list(
            __supported_data_files__.keys())
        if not test:
            if comm.rank == 0:
                for f in list(__supported_data_files__.keys()):
                    to_write += [
                        '-- %s -- %s' %
                        (f, __supported_data_files__[f].extension)
                    ]

                to_write += [
                    '', "To get more info on a given file format, see",
                    ">> spyking-circus file_format -i"
                ]

                print_and_log(to_write, 'error', logger)
            sys.exit(0)

        try:
            self.parser.get('detection', 'radius')
        except Exception:
            self.parser.set('detection', 'radius', 'auto')
        try:
            self.parser.getint('detection', 'radius')
        except Exception:
            self.parser.set('detection', 'radius',
                            str(int(self.probe['radius'])))

        if self.parser.getboolean('triggers', 'clean_artefact'):
            if (self.parser.get('triggers',
                                'trig_file') == '') or (self.parser.get(
                                    'triggers', 'trig_windows') == ''):
                if comm.rank == 0:
                    print_and_log([
                        "trig_file and trig_windows must be specified in [triggers]"
                    ], 'error', logger)
                sys.exit(0)

        units = ['ms', 'timestep']
        test = self.parser.get('triggers', 'trig_unit').lower() in units
        if not test:
            if comm.rank == 0:
                print_and_log(
                    ["trig_unit in [triggers] should be in %s" % str(units)],
                    'error', logger)
            sys.exit(0)
        else:
            self.parser.set(
                'triggers', 'trig_in_ms',
                str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))

        self.parser.set(
            'triggers', 'trig_file',
            os.path.abspath(
                os.path.expanduser(self.parser.get('triggers', 'trig_file'))))
        self.parser.set(
            'triggers', 'trig_windows',
            os.path.abspath(
                os.path.expanduser(self.parser.get('triggers',
                                                   'trig_windows'))))

        units = ['ms', 'timestep']
        test = self.parser.get('triggers', 'dead_unit').lower() in units
        if not test:
            if comm.rank == 0:
                print_and_log(
                    ["dead_unit in [triggers] should be in %s" % str(units)],
                    'error', logger)
            sys.exit(0)
        else:
            self.parser.set(
                'triggers', 'dead_in_ms',
                str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))

        self.parser.set(
            'triggers', 'dead_file',
            os.path.abspath(
                os.path.expanduser(self.parser.get('triggers', 'dead_file'))))

        test = (self.parser.get('clustering', 'extraction').lower()
                in ['median-raw', 'median-pca', 'mean-raw', 'mean-pca'])
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "Only 4 extraction modes in [clustering]: median-raw, median-pca, mean-raw or mean-pca!"
                ], 'error', logger)
            sys.exit(0)

        test = (self.parser.get('detection', 'peaks').lower()
                in ['negative', 'positive', 'both'])
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "Only 3 detection modes for peaks in [detection]: negative, positive, both"
                ], 'error', logger)

        try:
            os.makedirs(f_next)
        except Exception:
            pass

        self.parser.set('data', 'data_file', self.file_name)

        if self.parser.get('data', 'output_dir') != '':
            path = os.path.abspath(
                os.path.expanduser(self.parser.get('data', 'output_dir')))
            self.parser.set('data', 'output_dir', path)
            file_out = os.path.join(path, os.path.basename(f_next))
            if not os.path.exists(file_out):
                os.makedirs(file_out)
        else:
            file_out = os.path.join(f_next, os.path.basename(f_next))

        self.parser.set('data', 'data_file_no_overwrite',
                        file_out + '_all_sc.dat')
        self.parser.set('data', 'file_out',
                        file_out)  # Output file without suffix
        self.parser.set(
            'data', 'file_out_suff', file_out +
            self.parser.get('data', 'suffix'))  # Output file with suffix
        self.parser.set('data', 'data_file_noext',
                        f_next)  # Data file (assuming .filtered at the end)

        is_cluster = check_if_cluster()

        self.parser.set('data', 'is_cluster', str(is_cluster))

        if is_cluster:
            print_and_log([
                "Cluster detected, so using local /tmp folders and blosc compression"
            ], 'debug', logger)
            self.parser.set('data', 'global_tmp', 'False')
            self.parser.set('data', 'blosc_compress', 'True')
        else:
            print_and_log(
                ["Cluster not detected, so using global /tmp folder"], 'debug',
                logger)

        for section in ['whitening', 'clustering']:
            test = (self.parser.getfloat(section, 'nb_elts') >
                    0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
            if not test:
                if comm.rank == 0:
                    print_and_log(
                        ["nb_elts in [%s] should be in [0,1]" % section],
                        'error', logger)
                sys.exit(0)

        test = (self.parser.getfloat('clustering', 'nclus_min') >=
                0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["nclus_min in [clustering] should be in [0,1["],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('clustering', 'noise_thr') >=
                0) and (self.parser.getfloat('clustering', 'noise_thr') <= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["noise_thr in [clustering] should be in [0,1]"],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('validating', 'test_size') >
                0) and (self.parser.getfloat('validating', 'test_size') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["test_size in [validating] should be in ]0,1["],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('clustering', 'cc_merge') >=
                0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["cc_merge in [validating] should be in [0,1]"],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('clustering', 'nclus_min') >=
                0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["nclus_min in [validating] should be in [0,1["],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('merging', 'auto_mode') >= 0)
        if not test:
            if comm.rank == 0:
                print_and_log(["auto_mode in [merging] should be positive"],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getint('detection', 'oversampling_factor') >= 0)
        if not test:
            if comm.rank == 0:
                print_and_log(
                    ["oversampling_factor in [detection] should be postiive["],
                    'error', logger)
            sys.exit(0)

        test = (not self.parser.getboolean('data', 'overwrite')
                and not self.parser.getboolean('filtering', 'filter'))
        if test:
            if comm.rank == 0:
                print_and_log(
                    ["If no filtering, then overwrite should be True"],
                    'error', logger)
            sys.exit(0)

        fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
        for section in ['clustering', 'validating', 'triggers']:
            test = self.parser.get('clustering',
                                   'make_plots').lower() in fileformats
            if not test:
                if comm.rank == 0:
                    print_and_log([
                        "make_plots in [%s] should be in %s" %
                        (section, str(fileformats))
                    ], 'error', logger)
                sys.exit(0)

        dispersion = self.parser.get('clustering', 'dispersion').replace(
            '(', '').replace(')', '').split(',')
        dispersion = list(map(float, dispersion))
        test = (0 < dispersion[0]) and (0 < dispersion[1])
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "min and max dispersions in [clustering] should be positive"
                ], 'error', logger)
            sys.exit(0)

        pcs_export = ['prompt', 'none', 'all', 'some']
        test = self.parser.get('converting',
                               'export_pcs').lower() in pcs_export
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "export_pcs in [converting] should be in %s" %
                    str(pcs_export)
                ], 'error', logger)
            sys.exit(0)
        else:
            if self.parser.get('converting', 'export_pcs').lower() == 'none':
                self.parser.set('converting', 'export_pcs', 'n')
            elif self.parser.get('converting', 'export_pcs').lower() == 'some':
                self.parser.set('converting', 'export_pcs', 's')
            elif self.parser.get('converting', 'export_pcs').lower() == 'all':
                self.parser.set('converting', 'export_pcs', 'a')
示例#22
0
    def __init__(self, file_name, create_folders=True, **kwargs):
        """
        Parameters:
        ----------
        file_name : string
            a path containing the *params file.

        create_folders : bool
            if a folder will be created. If true, output_dir in the data section
        of the param file will be created.

        
        Returns:
        --------
        a CircusParser object. 

        """
        self.file_name = os.path.abspath(file_name)
        f_next, extension = os.path.splitext(self.file_name)
        file_path = os.path.dirname(self.file_name)
        self.file_params = f_next + '.params'
        self.do_folders = create_folders
        self.parser = configparser.ConfigParser()

        valid_path = check_valid_path(self.file_params)
        if not valid_path:
            print_and_log(
                ["Not all nodes can read/write the data file. Check path?"],
                'error', logger)
            sys.exit(0)

        ## First, we remove all tabulations from the parameter file, in order
        ## to secure the parser
        if comm.rank == 0:
            myfile = open(self.file_params, 'r')
            lines = myfile.readlines()
            myfile.close()
            myfile = open(self.file_params, 'w')
            for l in lines:
                myfile.write(l.replace('\t', ''))
            myfile.close()

        comm.Barrier()

        self._N_t = None

        if not os.path.exists(self.file_params):
            if comm.rank == 0:
                print_and_log(["%s does not exist" % self.file_params],
                              'error', logger)
            sys.exit(0)

        if comm.rank == 0:
            print_and_log(
                ['Creating a Circus Parser for datafile %s' % self.file_name],
                'debug', logger)
        self.parser.read(self.file_params)

        for section in self.__all_sections__:
            if self.parser.has_section(section):
                for (key, value) in self.parser.items(section):
                    self.parser.set(section, key, value.split('#')[0].rstrip())
            else:
                self.parser.add_section(section)

        for item in self.__default_values__ + self.__extra_values__:
            section, name, val_type, value = item
            try:
                if val_type is 'bool':
                    self.parser.getboolean(section, name)
                elif val_type is 'int':
                    self.parser.getint(section, name)
                elif val_type is 'float':
                    self.parser.getfloat(section, name)
                elif val_type is 'string':
                    self.parser.get(section, name)
            except Exception:
                self.parser.set(section, name, value)

        for key, value in kwargs.items():
            for section in self.__all_sections__:
                if self.parser._sections[section].has_key(key):
                    self.parser._sections[section][key] = value

        if self.do_folders and self.parser.get('data', 'output_dir') == '':
            try:
                os.makedirs(f_next)
            except Exception:
                pass

        self.parser.set('data', 'data_file', self.file_name)

        if self.parser.get('data', 'output_dir') != '':
            path = os.path.abspath(
                os.path.expanduser(self.parser.get('data', 'output_dir')))
            self.parser.set('data', 'output_dir', path)
            file_out = os.path.join(path, os.path.basename(f_next))
            if not os.path.exists(file_out) and self.do_folders:
                os.makedirs(file_out)
            self.logfile = file_out + '.log'
        else:
            file_out = os.path.join(f_next, os.path.basename(f_next))
            self.logfile = f_next + '.log'

        self.parser.set('data', 'data_file_no_overwrite',
                        file_out + '_all_sc.dat')
        self.parser.set('data', 'file_out',
                        file_out)  # Output file without suffix
        self.parser.set(
            'data', 'file_out_suff', file_out +
            self.parser.get('data', 'suffix'))  # Output file with suffix
        self.parser.set('data', 'data_file_noext',
                        f_next)  # Data file (assuming .filtered at the end)

        # read .prb file
        try:
            self.parser.get('detection', 'radius')
        except Exception:  # radius == auto by default
            self.parser.set('detection', 'radius', 'auto')

        try:
            self.parser.getint('detection', 'radius')
        except Exception:  # when radius = auto in params file
            self.probe = read_probe(self.parser, radius_in_probe=True)
            self.parser.set('detection', 'radius',
                            str(int(self.probe['radius'])))
        else:
            self.probe = read_probe(self.parser, radius_in_probe=False)

        dead_channels = self.parser.get('detection', 'dead_channels')
        if dead_channels != '':
            dead_channels = parse_dead_channels(dead_channels)
            if comm.rank == 0:
                print_and_log(
                    ["Removing dead channels %s" % str(dead_channels)],
                    'debug', logger)
            for key in dead_channels.keys():
                if key in self.probe["channel_groups"].keys():
                    for channel in dead_channels[key]:
                        n_before = len(
                            self.probe["channel_groups"][key]['channels'])
                        self.probe["channel_groups"][key]['channels'] = list(
                            set(self.probe["channel_groups"][key]
                                ['channels']).difference(dead_channels[key]))
                        n_after = len(
                            self.probe["channel_groups"][key]['channels'])
                else:
                    if comm.rank == 0:
                        print_and_log([
                            "Probe has no group named %s for dead channels" %
                            key
                        ], 'debug', logger)

        N_e = 0
        for key in self.probe['channel_groups'].keys():
            N_e += len(self.probe['channel_groups'][key]['channels'])

        self.set('data', 'N_e', str(N_e))
        self.set('data', 'N_total', str(self.probe['total_nb_channels']))
        self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
        self.nb_channels = self.probe['total_nb_channels']

        if N_e > self.nb_channels:
            if comm.rank == 0:
                print_and_log([
                    'The number of analyzed channels is higher than the number of recorded channels'
                ], 'error', logger)
            sys.exit(0)

        if N_e == 1 and self.parser.getboolean('filtering', 'remove_median'):
            if comm.rank == 0:
                print_and_log([
                    "With 1 channel, remove_median in [filtering] is not possible"
                ], 'error', logger)
            sys.exit(0)

        to_write = [
            "You must specify explicitly the file format in the config file",
            "Please have a look to the documentation and add a file_format",
            "parameter in the [data] section. Valid files formats can be:", ''
        ]
        try:
            self.file_format = self.parser.get('data', 'file_format')
        except Exception:
            if comm.rank == 0:
                for f in __supported_data_files__.keys():
                    to_write += [
                        '-- %s -- %s' %
                        (f, __supported_data_files__[f].extension)
                    ]

                to_write += [
                    '', "To get more info on a given file format, see",
                    ">> spyking-circus file_format -i"
                ]

                print_and_log(to_write, 'error', logger)
            sys.exit(0)

        test = self.file_format.lower() in __supported_data_files__.keys()
        if not test:
            if comm.rank == 0:
                for f in __supported_data_files__.keys():
                    to_write += [
                        '-- %s -- %s' %
                        (f, __supported_data_files__[f].extension)
                    ]

                to_write += [
                    '', "To get more info on a given file format, see",
                    ">> spyking-circus file_format -i"
                ]

                print_and_log(to_write, 'error', logger)
            sys.exit(0)

        try:
            self.parser.get('detection', 'radius')
        except Exception:
            self.parser.set('detection', 'radius', 'auto')
        try:
            self.parser.getint('detection', 'radius')
        except Exception:
            self.parser.set('detection', 'radius',
                            str(int(self.probe['radius'])))

        if self.parser.getboolean('triggers', 'clean_artefact'):
            if (self.parser.get('triggers',
                                'trig_file') == '') or (self.parser.get(
                                    'triggers', 'trig_windows') == ''):
                if comm.rank == 0:
                    print_and_log([
                        "trig_file and trig_windows must be specified in [triggers]"
                    ], 'error', logger)
                sys.exit(0)

        units = ['ms', 'timestep']
        test = self.parser.get('triggers', 'trig_unit').lower() in units
        if not test:
            if comm.rank == 0:
                print_and_log(
                    ["trig_unit in [triggers] should be in %s" % str(units)],
                    'error', logger)
            sys.exit(0)
        else:
            self.parser.set(
                'triggers', 'trig_in_ms',
                str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))

        if self.parser.getboolean('triggers', 'clean_artefact'):
            for key in ['trig_file', 'trig_windows']:
                myfile = os.path.abspath(
                    os.path.expanduser(self.parser.get('triggers', key)))
                if not os.path.exists(myfile):
                    if comm.rank == 0:
                        print_and_log(
                            ["File %s can not be found" % str(myfile)],
                            'error', logger)
                    sys.exit(0)
                self.parser.set('triggers', key, myfile)

        units = ['ms', 'timestep']
        test = self.parser.get('triggers', 'dead_unit').lower() in units
        if not test:
            if comm.rank == 0:
                print_and_log(
                    ["dead_unit in [triggers] should be in %s" % str(units)],
                    'error', logger)
            sys.exit(0)
        else:
            self.parser.set(
                'triggers', 'dead_in_ms',
                str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))

        if self.parser.getboolean('triggers', 'ignore_times'):
            myfile = os.path.abspath(
                os.path.expanduser(self.parser.get('triggers', 'dead_file')))
            if not os.path.exists(myfile):
                if comm.rank == 0:
                    print_and_log(["File %s can not be found" % str(myfile)],
                                  'error', logger)
                sys.exit(0)
            self.parser.set('triggers', 'dead_file', myfile)

        test = (self.parser.get('clustering', 'extraction').lower()
                in ['median-raw', 'median-pca', 'mean-raw', 'mean-pca'])
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "Only 4 extraction modes in [clustering]: median-raw, median-pca, mean-raw or mean-pca!"
                ], 'error', logger)
            sys.exit(0)

        test = (self.parser.get('detection', 'peaks').lower()
                in ['negative', 'positive', 'both'])
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "Only 3 detection modes for peaks in [detection]: negative, positive, both"
                ], 'error', logger)
            sys.exit(0)

        common_ground = self.parser.get('filtering', 'common_ground')
        if common_ground != '':
            try:
                self.parser.set('filtering', 'common_ground',
                                str(int(common_ground)))
            except Exception:
                self.parser.set('filtering', 'common_ground', '-1')
        else:
            self.parser.set('filtering', 'common_ground', '-1')

        common_ground = self.parser.getint('filtering', 'common_ground')

        all_electrodes = []
        for key in self.probe['channel_groups'].keys():
            all_electrodes += self.probe['channel_groups'][key]['channels']

        test = (common_ground == -1) or common_ground in all_electrodes
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "Common ground in filtering section should be a valid electrode"
                ], 'error', logger)
            sys.exit(0)

        is_cluster = check_if_cluster()

        self.parser.set('data', 'is_cluster', str(is_cluster))

        if is_cluster:
            print_and_log([
                "Cluster detected, so using local /tmp folders and blosc compression"
            ], 'debug', logger)
            self.parser.set('data', 'global_tmp', 'False')
            self.parser.set('data', 'blosc_compress', 'True')
        else:
            print_and_log(
                ["Cluster not detected, so using global /tmp folder"], 'debug',
                logger)

        for section in ['whitening', 'clustering']:
            test = (self.parser.getfloat(section, 'nb_elts') >
                    0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
            if not test:
                if comm.rank == 0:
                    print_and_log(
                        ["nb_elts in [%s] should be in [0,1]" % section],
                        'error', logger)
                sys.exit(0)

        test = (self.parser.getfloat('clustering', 'nclus_min') >=
                0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["nclus_min in [clustering] should be in [0,1["],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('clustering', 'noise_thr') >=
                0) and (self.parser.getfloat('clustering', 'noise_thr') <= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["noise_thr in [clustering] should be in [0,1]"],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('validating', 'test_size') >
                0) and (self.parser.getfloat('validating', 'test_size') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["test_size in [validating] should be in ]0,1["],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('clustering', 'cc_merge') >=
                0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["cc_merge in [validating] should be in [0,1]"],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('clustering', 'nclus_min') >=
                0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["nclus_min in [validating] should be in [0,1["],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('data', 'memory_usage') >
                0) and (self.parser.getfloat('data', 'memory_usage') <= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["memory_usage in [data] should be in ]0,1]"],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('merging', 'auto_mode') >=
                0) and (self.parser.getfloat('merging', 'auto_mode') <= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["auto_mode in [merging] should be in [0, 1]"],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('merging', 'noise_limit') >= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["noise_limit in [merging] should be > 1"],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getfloat('merging', 'sparsity_limit') <= 1)
        if not test:
            if comm.rank == 0:
                print_and_log(["sparsity_limit in [merging] should be < 1"],
                              'error', logger)
            sys.exit(0)

        test = (self.parser.getint('detection', 'oversampling_factor') >= 0)
        if not test:
            if comm.rank == 0:
                print_and_log(
                    ["oversampling_factor in [detection] should be positive["],
                    'error', logger)
            sys.exit(0)

        test = (not self.parser.getboolean('data', 'overwrite')
                and not self.parser.getboolean('filtering', 'filter'))
        if test:
            if comm.rank == 0:
                print_and_log(
                    ["If no filtering, then overwrite should be True"],
                    'error', logger)
            sys.exit(0)

        fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
        for section in ['clustering', 'validating', 'triggers']:
            test = self.parser.get('clustering',
                                   'make_plots').lower() in fileformats
            if not test:
                if comm.rank == 0:
                    print_and_log([
                        "make_plots in [%s] should be in %s" %
                        (section, str(fileformats))
                    ], 'error', logger)
                sys.exit(0)

        fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
        for section in ['clustering']:
            test = self.parser.get('clustering',
                                   'debug_plots').lower() in fileformats
            if not test:
                if comm.rank == 0:
                    print_and_log([
                        "debug_plots in [%s] should be in %s" %
                        (section, str(fileformats))
                    ], 'error', logger)
                sys.exit(0)

        methods = [
            'distance', 'dip', 'folding', 'nd-folding', 'bhatta', 'nd-bhatta'
        ]
        test = self.parser.get('clustering',
                               'merging_method').lower() in methods
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "merging_method in [%s] should be in %s" %
                    (section, str(methods))
                ], 'error', logger)
            sys.exit(0)

        if self.parser.get('clustering', 'merging_param').lower() == 'default':
            method = self.parser.get('clustering', 'merging_method').lower()
            if method == 'dip':
                self.parser.set('clustering', 'merging_param', '0.5')
            elif method == 'distance':
                self.parser.set('clustering', 'merging_param', '3')
            elif method in ['folding', 'nd-folding']:
                self.parser.set('clustering', 'merging_param', '1e-9')
            elif method in ['bhatta', 'nd-bhatta']:
                self.parser.set('clustering', 'merging_param', '2')

        has_same_elec = self.parser.has_option('clustering', 'sim_same_elec')
        has_dip_thresh = self.parser.has_option('clustering', 'dip_threshold')

        if has_dip_thresh:
            dip_threshold = self.parser.getfloat('clustering', 'dip_threshold')

        if has_dip_thresh and (dip_threshold > 0):
            if comm.rank == 0:
                print_and_log([
                    "dip_threshold in [clustering] is deprecated since 0.8.4",
                    "and you should now use merging_method and merging_param",
                    "Please upgrade your parameter file to a more recent version",
                    "By default a distance merging method with param 3 is assumed"
                ], 'info', logger)
            self.parser.set('clustering', 'merging_param', str(3))
            self.parser.set('clustering', 'merging_method', 'distance')
        elif has_same_elec:
            sim_same_elec = self.parser.get('clustering', 'sim_same_elec')
            if comm.rank == 0:
                print_and_log([
                    "sim_same_elec in [clustering] is deprecated since 0.8.4",
                    "and you should now use merging_method and merging_param",
                    "Please upgrade your parameter file to a more recent version",
                    "Meanwhile a distance merging method with param %s is assumed"
                    % sim_same_elec
                ], 'info', logger)
            self.parser.set('clustering', 'merging_param', sim_same_elec)
            self.parser.set('clustering', 'merging_method', 'distance')

        dispersion = self.parser.get('clustering', 'dispersion').replace(
            '(', '').replace(')', '').split(',')
        dispersion = map(float, dispersion)
        test = (0 < dispersion[0]) and (0 < dispersion[1])
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "min and max dispersions in [clustering] should be positive"
                ], 'error', logger)
            sys.exit(0)

        pcs_export = ['prompt', 'none', 'all', 'some']
        test = self.parser.get('converting',
                               'export_pcs').lower() in pcs_export
        if not test:
            if comm.rank == 0:
                print_and_log([
                    "export_pcs in [converting] should be in %s" %
                    str(pcs_export)
                ], 'error', logger)
            sys.exit(0)
        else:
            if self.parser.get('converting', 'export_pcs').lower() == 'none':
                self.parser.set('converting', 'export_pcs', 'n')
            elif self.parser.get('converting', 'export_pcs').lower() == 'some':
                self.parser.set('converting', 'export_pcs', 's')
            elif self.parser.get('converting', 'export_pcs').lower() == 'all':
                self.parser.set('converting', 'export_pcs', 'a')

        if self.parser.getboolean('detection', 'hanning'):
            if comm.rank == 0:
                print_and_log(["Hanning filtering is activated"], 'debug',
                              logger)
示例#23
0
def apply_patch_for_similarities(params, extension):

    if not test_patch_for_similarities(params, extension):

        file_out_suff = params.get('data', 'file_out_suff')

        if comm.rank == 0:

            print_and_log(["Fixing overlaps from 0.5.XX..."], 'default',
                          logger)

            over_file = file_out_suff + '.overlap.hdf5'
            if os.path.exists(over_file):
                over_x = h5py.File(over_file, 'r',
                                   libver='latest').get('over_x')[:].ravel()
                over_y = h5py.File(over_file, 'r',
                                   libver='latest').get('over_y')[:].ravel()
                over_data = h5py.File(
                    over_file, 'r',
                    libver='latest').get('over_data')[:].ravel()
                over_shape = h5py.File(
                    over_file, 'r',
                    libver='latest').get('over_shape')[:].ravel()
                overlap = scipy.sparse.csc_matrix(
                    (over_data, (over_x, over_y)), shape=over_shape)
            else:
                raise Exception('No overlaps found! Check suffix?')

            myfile2 = h5py.File(file_out_suff +
                                '.templates%s.hdf5' % extension,
                                'r+',
                                libver='latest')
            N_tm = int(numpy.sqrt(overlap.shape[0]))
            N_t = params.getint('detection', 'N_t')

            if 'maxoverlap' in myfile2.keys():
                maxoverlap = myfile2['maxoverlap']
            else:
                maxoverlap = myfile2.create_dataset('maxoverlap',
                                                    shape=(N_tm // 2,
                                                           N_tm // 2),
                                                    dtype=numpy.float32)

            if 'maxlag' in myfile2.keys():
                maxlag = myfile2['maxlag']
            else:
                maxlag = myfile2.create_dataset('maxlag',
                                                shape=(N_tm // 2, N_tm // 2),
                                                dtype=numpy.int32)

            if 'version' in myfile2.keys():
                version = myfile2['version']
            else:
                version = myfile2.create_dataset(
                    'version',
                    data=numpy.array(
                        [circus.__version__.encode('ascii', 'ignore')]))

            for i in get_tqdm_progressbar(xrange(N_tm // 2 - 1)):
                data = overlap[i * N_tm + i + 1:i * N_tm + N_tm // 2].toarray()
                maxlag[i, i + 1:] = N_t - numpy.argmax(data, 1)
                maxlag[i + 1:, i] = -maxlag[i, i + 1:]
                maxoverlap[i, i + 1:] = numpy.max(data, 1)
                maxoverlap[i + 1:, i] = maxoverlap[i, i + 1:]
            myfile2.close()
示例#24
0
    def _update_rate_values(self):
        """
        Updates the values in sampling points of the following values:

        - template width (N_t) in [detection] 
        - minimal distance between peaks (dist_peaks) in [detection]
        - the template shift (template_shift) in [detection]
        - the jitter range (jitter_range) in [detection]
        - the (savgol_window) in [clustering]
        - the (chunk_size) in [data, whitening, fitting]
        - the (safety_time) in [clustering, whitening, extracting]
        - the (refractory) in [fitting]
        """

        if self._N_t is None:

            if comm.rank == 0:
                print_and_log(
                    ['Changing all values in the param depending on the rate'],
                    'debug', logger)

            spike_width = self.getfloat('detection', 'spike_width')
            self._N_t = self.getfloat('detection', 'N_t')

            # template width from milisecond to sampling points
            self._N_t = int(self.rate * self._N_t * 1e-3)
            if numpy.mod(self._N_t, 2) == 0:
                self._N_t += 1
            self.set('detection', 'N_t', self._N_t)
            self.set('detection', 'dist_peaks', self._N_t)
            self.set('detection', 'template_shift', (self._N_t - 1) // 2)
            self.set('detection', 'spike_width',
                     int(self.rate * spike_width * 1e-3))

            # jitter_range form milisecond sampling points
            jitter = self.getfloat('detection', 'jitter_range')
            jitter_range = int(self.rate * jitter * 1e-3)
            self.set('detection', 'jitter_range', jitter_range)

            # savgol from milisecond to sampling points
            self._savgol = int(self.rate * 0.5 * 1e-3)
            if numpy.mod(self._savgol, 2) == 0:
                self._savgol += 1

            self.set('clustering', 'savgol_window', self._savgol)

            if self.parser._sections['fitting'].has_key('chunk'):
                self.parser.set('fitting', 'chunk_size',
                                self.parser._sections['fitting']['chunk'])

            # chunck_size from second to sampling points
            for section in ['data', 'whitening', 'fitting']:
                chunk = self.parser.getfloat(section, 'chunk_size')
                chunk_size = int(chunk * self.rate)
                self.set(section, 'chunk_size', chunk_size)

            # safety_time from milisecond to sampling points
            for section in ['clustering', 'whitening', 'extracting']:
                safety_time = self.get(section, 'safety_time')
                if safety_time == 'auto':
                    self.set(section, 'safety_time', self._N_t // 3)
                else:
                    safety_time = int(float(safety_time) * self.rate * 1e-3)
                    self.set(section, 'safety_time', safety_time)

            # refractory from milisecond to sampling points
            refractory = self.getfloat('fitting', 'refractory')
            self.set('fitting', 'refractory',
                     int(refractory * self.rate * 1e-3))