Exemplo n.º 1
0
def main():
    parser = OptionParser()

    parser.add_option("--db", default='flydra_db', help="FlydraDB directory")

    parser.add_option("--nocache", help="Ignores already computed results.",
                      default=False, action="store_true")    
    
    parser.add_option("--sigma", help="Kernel spread (degrees)",
                      type="float", default=6)
   
    parser.add_option("--source", default='luminance', help="Source table")
    parser.add_option("--target", default='contrast', help="Destination table")
    
    (options, args) = parser.parse_args()

    kernel = get_contrast_kernel(sigma_deg=options.sigma, eyes_interact=False)
    kernel = kernel.astype('float32').copy('C')
    
    db = FlydraDB(options.db, False)
    
    if args:
        do_samples = args
    else:
        do_samples = db.list_samples()
        do_samples = filter(lambda x: db.has_table(x, options.source),
                            do_samples)
        
    if not do_samples:
        raise Exception('No samples with table "%s" found. ' % options.source)
    
    for i, sample_id in enumerate(do_samples):
        
        logger.info('Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id))
        
        if not db.has_sample(sample_id):
            raise Exception('Sample %s not found in db.' % sample_id)
        
        if not db.has_table(sample_id, options.source):
            raise Exception('Sample %s does not have table %s; skipping.' \
                            % (sample_id, options.source))
        
        if db.has_table(sample_id, options.target) and not options.nocache:
            logger.info('Already computed "%s" for %s; skipping' % \
                (options.target, sample_id))
            continue

        luminance = db.get_table(sample_id, options.source)
    
        contrast = compute_contrast_for_table(luminance, kernel)
        
        db.set_table(sample_id, options.target, contrast)
        
        db.release_table(luminance)
        
    db.close()
def main():
    parser = OptionParser(usage=description)
    parser.add_option("--saccade_data", help="Main data directory",
                      default='saccade_data')
    parser.add_option("--db", help='Location of output Flydra db.')
        
    (options, args) = parser.parse_args() #@UnusedVariable
    
    if not options.db:
        raise Exception('Please define FlydraDB directory using `--db`.')
    
    verbose = True
    
    flydra_db = FlydraDB(options.db, create=True)
    
    matlab_dir = options.saccade_data
    for group in os.listdir(matlab_dir):
        group_dir = os.path.join(matlab_dir, group)
        if not os.path.isdir(group_dir):                
            continue
        
        if verbose:
            print("Opening {0}".format(group))
        
        for file in [file for file in os.listdir(group_dir) 
            if (file.startswith('magno_')) \
               and file.endswith('.mat')]:
            
            sample = file[file.index('_') + 1:file.index('.')]
            
            if verbose:
                print("  - Considering sample {0}".format(sample.__repr__()))
        
            if not flydra_db.has_sample(sample):
                flydra_db.add_sample(sample)
            flydra_db.add_sample_to_group(sample, group)
#           flydra_db.add_sample_to_group(sample, 'ros')
            
            filename = os.path.join(group_dir, file)
            
            exp_data, attributes = read_raw_data(filename)
            
            consider_importing_processed(flydra_db, sample, exp_data, attributes)
            
            flydra_db.set_attr(sample, 'species', attributes['species'])
            flydra_db.set_attr(sample, 'background', attributes['background'])            
            flydra_db.set_table(sample, EXP_DATA_TABLE, exp_data)
            
    flydra_db.close()
Exemplo n.º 3
0
class FlydraImage(Generator):
    ''' This block outputs the retinal images from a FlydraDB for 
        a particular sample. '''
    Block.alias('flydra_db_image')
        
    Block.config('db', 'FlydraDB database directory')
    Block.config('sample', 'Sample ID -- such as "DATA20080611_191809".')
    Block.config('image', 'Which retinal image to display.')
    
    Block.output('obj_id')
    Block.output('frame')
    Block.output('image')
    
    def init(self):
        self.db = FlydraDB(self.config.db)
        if not self.db.has_sample(self.config.sample):
            raise ValueError('Sample "%s" not found.' % self.config.sample)
        if not self.db.has_table(self.config.sample, self.config.image):
            raise ValueError('Table "%s" not found for sample %s.' % \
                    (self.config.image, self.config.sample))

        self.data = self.db.get_table(self.config.sample, self.config.image)
        
        self.next_index = 0
        if len(self.data) == 0:
            self.info('Empty rows for sample %s.' % self.config.sample)
            self.next_index = None

    def update(self):
        row = self.data[self.next_index]
        t = row['time']
        
        for field in ['obj_id', 'frame']:
            self.set_output(field, value=row[field], timestamp=t)
        
        self.set_output('image', row['value'], timestamp=t)
        
        self.next_index += 1
        if self.next_index == len(self.data):
            self.next_index = None
        
    def next_data_status(self):
        # TODO: put new interface
        if self.next_index is None: # EOF
            return (False, None)
        else:
            return (True, self.data[self.next_index]['time'])
Exemplo n.º 4
0
def compute_mean_generic(db, samples, image, operator):
    ''' 
    db: FlydraDB directory
    samples: list of IDs
    '''
    db = FlydraDB(db, False)
    
    results = { 'samples': {} }
    
    ex = Expectation()
    
    for i, id in enumerate(samples):
        progress('Computing mean %s' % image,
                 (i, len(samples)), "Sample %s" % id)
    
        if not (db.has_sample(id) and db.has_table(id, image)):
            raise ValueError('No table "%s" for id %s' % (image, id))
        
        data = db.get_table(id, image)
        
        values = data[:]['value']
        
        this = operator(values)
        
        # print "id: %s   len: %d  %d" % (id, len(data), len(values))
        ex.update(this, len(data))
    
        results['samples'][id] = this
            
        db.release_table(data)

    results['all'] = ex.get_value()
        
    db.close()
    
    return results 
def main():
    
    parser = OptionParser()
    
    parser.add_option("--db", default='flydra_db', help="FlydraDB directory")

    parser.add_option("--nocache", help="Ignores already computed results.",
                      default=False, action="store_true")

    parser.add_option("--compute_mu", help="Computes mu and optic flow.",
                      default=False, action="store_true")
    
    parser.add_option("--white", help="Computes luminance_w, with the arena"
                      " painted white.", default=False, action="store_true")
    
    parser.add_option("--host", help="Use a remote rfsee. Otherwise, use local process.",
                       default=None)
    
    (options, args) = parser.parse_args() #@UnusedVariable
     
        
    db = FlydraDB(options.db, False)
    
        
    # look for samples with the rows table
    do_samples = db.list_samples()
    do_samples = filter(lambda x: db.has_rows(x) and 
                        db.get_attr(x, 'stimulus') == 'nopost',
                        do_samples)
    if not do_samples:
        raise Exception('Cannot find samples to hallucinate about.')
        
    print "Summary, including nopost."
    for s in sorted(get_db_stimulus_stats(db, include_nopost=True),
                       key=(lambda x:-x.total_length)):
        print "stimulus: {s.stimulus:>10}  samples: {s.total_number:>5}  "\
              " total length: {len:>5} minutes".format(s=s, len=s.total_length / (60 * 60))
        
    
      
    stimulus_to_use = list(get_stimulus_to_use(db, len(do_samples)))
    
    for i, sample in enumerate(do_samples):
        stimulus = stimulus_to_use[i][0]
        print sample, stimulus 
    
    if options.white:
        target = 'hluminance_w'
    else:
        target = 'hluminance'
    
    for i, sample_id in enumerate(do_samples):
        stimulus = stimulus_to_use[i][0]
        stimulus_xml = stimulus_to_use[i][1]
            
        print 'Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id)
        
        if not db.has_sample(sample_id):
            raise Exception('Sample %s not found in db.' % sample_id)
        
        if not db.has_rows(sample_id):
            raise Exception('Sample %s does not have rows table.' % sample_id)
         
        if options.compute_mu:
            if db.has_table(sample_id, 'nearness') and not options.nocache:
                logger.info('Already computed nearness for %s; skipping' % sample_id)
                continue
        else:
            if db.has_table(sample_id, target) and not options.nocache:
                logger.info('Already computed luminance for %s; skipping' % sample_id)
                continue
        
        rows = db.get_rows(sample_id)
         
        results = render(rows, stimulus_xml, host=options.host,
                         compute_mu=options.compute_mu, white=options.white)
   
        db.set_table(sample_id, target, results['luminance'])
        
        if options.compute_mu:
            db.set_table(sample_id, 'hnearness', results['nearness'])
            db.set_table(sample_id, 'hretinal_velocities',
                         results['retinal_velocities'])
        
        db.release_table(rows)    
    
    db.close()
Exemplo n.º 6
0
def main():

    parser = OptionParser(usage=description)

    parser.add_option(
        "--db", default='flydra_db_directory', help="FlydraDB directory")
    parser.add_option("--model", help="ProcGraph model name.")
    parser.add_option(
        "--needs",
        help="Comma-separated list of tables required",
        default="rows,luminance")

    parser.add_option(
        "--interactive",
        help="Start compmake interactive session."
        " Otherwise run in batch mode",
        default=False,
        action="store_true")

    (options, args) = parser.parse_args()

    if options.model is None:
        print "Please specify the model."
        sys.exit(-3)

    print("Using FlydraDB directory %r." % options.db)
    db = FlydraDB(options.db, False)

    # TODO: make the storage inside options.db?
    set_namespace('run_pg_model_%s' % options.model)
    tables = options.needs.split(',')

    if args:
        samples = args
        for sample in samples:
            if not db.has_sample(sample):
                raise Exception('Unknown sample %r' % sample)
    else:
        samples = db.list_samples()

    if not samples:
        print 'No samples found'

    num_ok = 0
    for id in samples:
        enough = all(map(lambda t: db.has_table(id, t), tables))

        if not enough:
            continue

        num_ok += 1
        config = {'sample': id, 'db': options.db}
        comp(pg, options.model, config, job_id=id)

    logger.info(
        "Found %d/%d samples with tables %s." % (num_ok, len(samples), tables))

    if options.interactive:
        # start interactive session
        compmake_console()
    else:
        # batch mode
        # try to do everything
        batch_command('make all')
        # start the console if we are not done
        # (that is, make all failed for some reason)
        todo = list(parse_job_list('todo'))
        if todo:
            logger.info('Still %d jobs to do.' % len(todo))
            sys.exit(-2)
Exemplo n.º 7
0
class SamplesDB:
    
    def __init__(self, data, verbose=False):
        ''' data: base directory '''
        
        if not os.path.exists(data) or not os.path.isdir(data):
            raise Exception('Could not open directory %s' % data)
        
        self.data = data
        
        # self.use_cache = True
        self.use_cache = False
        self.use_flydra_db = True
         
        
        if self.use_cache:
            self.open_shelve()
        
        if self.use_flydra_db:
            self.open_flydra_db()
            
        
        self.groups = {}
        
        #self.group2samples = {}
        # maps id to .mat file
        self.sample2expmat = {}
        # maps id to .pickle file
        self.sample2exppickle = {}
        # list of all configurations
        self.configurations = set()
        # maps sample -> group
        self.sample2group = {}
        
        #print "Loading data in %s" % data
        
        for group in os.listdir(data):
            group_dir = os.path.join(data, group)
            if not os.path.isdir(group_dir):                
                continue
            
            # print "Reading group %s" % group
            group_record = Group()
            
            for file in [file for file in os.listdir(group_dir) 
                if file.startswith('data_') and file.endswith('.mat')]:
                id = file[5:-4]
                group_record.samples.add(id)
                self.sample2expmat[id] =  os.path.join(group_dir,file)
                self.sample2group[id] = group
                
                
            for file in [file for file in os.listdir(group_dir) 
                if file.startswith('data_') and file.endswith('.pickle')]: 
                id = file[5:-7]
                group_record.samples.add(id)
                self.sample2exppickle[id] = os.path.join(group_dir,file)
                self.sample2group[id] = group

            group_record.has_experimental_data = len(group_record.samples) > 0
            
            
            processed_dir = os.path.join(group_dir, 'processed')
            if not os.path.exists(processed_dir):
                if verbose:
                    print "No processed data found for %s." % group
                pass
                
            else:
                for conf in os.listdir(processed_dir):                
                    saccades = os.path.join(processed_dir, conf, 'saccades.mat')
                    if os.path.exists(saccades): 
                        group_record.configurations[conf] = saccades
                        # add to general list
                        self.configurations.add(conf)
#                    else:
#                        conf_dir = os.path.join(processed_dir, conf)
#                        for file in [file for file in os.listdir(conf_dir) 
#                            if file.startswith('processed_data_') and file.endswith('.mat')]: 
#                                  id = file[5:-7]

                # if we don't have exp data, get list of samples from
                # processed data
                if group_record.configurations and \
                    not group_record.has_experimental_data:
                    saccades = saccades_read_mat(saccades)
                    group_record.samples = set(numpy.unique(saccades['sample']))
                    for sample in group_record.samples:
                        self.sample2group[sample] = group

            if len(group_record.samples)> 0:
                self.groups[group] = group_record
                    
                print "has it", group, group_record.has_experimental_data
            
    def open_shelve(self):
        shelve_fname = os.path.join(self.data, 'shelve')
        self.shelve = shelve.open(shelve_fname, protocol=pickle.HIGHEST_PROTOCOL)
    
    def open_flydra_db(self):
        self.flydra_db = FlydraDB(os.path.join(self.data, 'sac_flydra_db'))
        
    def list_groups(self):
        """ Returns a list of the groups. """
        return natsorted(list(self.groups.keys()))
    
    def list_all_samples(self):
        """ Returns a list of all samples for all groups. """
        return natsorted(list(self.sample2group.keys()))
    
    def list_samples(self, group):
        """ Lists the samples in the given group. """
        return natsorted(list(self.groups[group].samples))
    
    def list_all_configurations(self):
        """ Lists all the configurations present in the data. """
        return natsorted(self.configurations)
        
    def list_configurations(self, group):
        """ Lists the configurations for the given group. """
        return natsorted(list(self.groups[group].configurations.keys()))
    
    def get_group_for_sample(self, sample):
        """ Returns the sample associated to the group. """
        return self.sample2group[sample]
    
    def get_saccades_for_group(self, group, configuration):
        """ Returns the saccades for the given group and configuration. 
            If configuration is not passed, we use the default.
        """
        if self.use_flydra_db:
            table = 'groupsaccades_%s' % configuration
            if self.flydra_db.has_sample(group) and \
               self.flydra_db.has_table(group, table):
                t = self.flydra_db.get_table(group, table)
                #value = t.copy()
                value = t
                #self.flydra_db.release_table(t)
                return value 
            
        if self.use_cache:
            key = str(('get_saccades_for_group', group, configuration))
            if key in self.shelve:
                return self.shelve[key]
    
        filename = self.groups[group].configurations[configuration]
        saccades = saccades_read_mat(filename)
        
        
        if self.use_flydra_db:
            if not self.flydra_db.has_sample(group):
                self.flydra_db.add_sample(group)
            self.flydra_db.set_table(group, table, saccades)
            
        if self.use_cache:
            self.shelve[key] = saccades
            
        return saccades
    
    def group_has_experimental_data(self, group):
        """ Returns true if this group has the raw orientation data.
            (mamarama has only saccades data. ) """
        return self.groups[group].has_experimental_data
    
    def has_experimental_data(self, sample):
        """ Returns true if this sample has the raw orientation data. """
        return sample in self.sample2expmat or sample in self. sample2exppickle

    def get_saccades_for_sample(self, sample, configuration):
        """ Returns the saccades for the given group and configuration. 
            If configuration is not passed, we use the default.
        """
        if self.use_flydra_db:
            table = 'saccades_%s' % configuration
            if self.flydra_db.has_sample(sample) and \
               self.flydra_db.has_table(sample, table):
                t = self.flydra_db.get_table(sample, table)
                #value =  t.copy()
                value = t
                #self.flydra_db.release_table(t)
                return value 
            
        if self.use_cache:
            key = str(('get_saccades_for_sample', sample, configuration))
            if key in self.shelve:
                return self.shelve[key]
    
        group = self.get_group_for_sample(sample)
        group_saccades  = self.get_saccades_for_group(group, configuration) 

        print group_saccades[0].dtype
#        with open('tmp.pickle','w') as f:
#            pickle.dump(f, group_saccades)        
        mine = group_saccades[:]['sample'] == sample
        
        saccades = group_saccades[mine]
        
        if len(saccades) == 0:
            raise Exception('No saccades found for %s' % sample)
        
        
        if self.use_flydra_db:
            if not self.flydra_db.has_sample(sample):
                self.flydra_db.add_sample(sample)

            self.flydra_db.set_table(sample, table, saccades)
            
        if self.use_cache:
            self.shelve[key] = saccades
        
        return saccades
        
    def get_experimental_data(self, sample):
#        if self.use_flydra_db:
#            table = 'tethered_data'
#            if self.flydra_db.has_sample(sample) and \
#               self.flydra_db.has_table(sample, table):
#                t = self.flydra_db.get_table(sample, table)
#                #value =  t.copy()
#                value = t
#                #self.flydra_db.release_table(t)
#                return value 
#        
        if self.use_cache:
            if sample in self.shelve:
                return self.shelve[sample]
                
        if sample in self.sample2expmat:
            data = scipy.io.loadmat(self.sample2expmat[sample], squeeze_me=True)
            data = data['data']
            # convert from array to hash
            assert isinstance(data, numpy.ndarray)
            data = dict(map(lambda field: (field, data[field]), data.dtype.fields))
            # convert from array to string
            for k in list(data.keys()):
                if data[k].dtype.char == 'U':
                    data[k] = str(data[k])
                
            # make sure everything is 1d array
            def as1d(x):  
                if x.dtype == 'object':
                    x = x.tolist()
                return x.reshape(len(x))
            data['exp_orientation'] = as1d(data['exp_orientation'])
            data['exp_timestamps'] = as1d(data['exp_timestamps'])
            
        elif sample in self.sample2exppickle:
            with open(self.sample2exppickle[sample], 'rb') as f:
                data = cPickle.load(f)      
            
        else:
            raise Exception('no data for sample %s found' % sample)
        
#        if self.use_flydra_db:
#            if not self.flydra_db.has_sample(sample):
#                self.flydra_db.add_sample(sample)
#
#            self.flydra_db.set_table(sample, table, data)
            
        if self.use_cache:
            self.shelve[sample] = data
        
        return data
    
    def __getstate__(self):
        # do not pickle the shelve
        all = dict(self.__dict__)
        all['shelve'] = None
        all['flydra_db'] = None
        return all
Exemplo n.º 8
0
def main():
    
    parser = OptionParser(usage=description)

    parser.add_option("--db", default='flydra_db', help="FlydraDB directory")

    parser.add_option("--nocache", help="Ignores already computed results.",
                      default=False, action="store_true")
    
    parser.add_option("--white", help="Computes luminance_w, with the arena"
                      " painted white.", default=False, action="store_true")
    
    parser.add_option("--host", help="Use a remote rfsee. Otherwise," 
                      "use local process.", default=None)
    
    (options, args) = parser.parse_args()
    

    if options.db is None:
        logger.error('Please specify a directory using --db.')
        sys.exit(-1)
        
        
    db = FlydraDB(options.db)
    
    if args:
        do_samples = args
        
    else:
        # look for samples with the rows table
        all_samples = db.list_samples()
        do_samples = filter(lambda x: db.has_saccades(x) and 
                                      db.has_attr(x, 'stimulus_xml'),
                            all_samples)
        logger.info('Found %d/%d samples with saccades and stimulus info.' % 
                    (len(do_samples), len(all_samples)))
    
    image = 'luminance_w' if options.white else 'luminance'
        
    target_start = 'saccades_view_start_%s' % image
    target_stop = 'saccades_view_stop_%s' % image
    target_rstop = 'saccades_view_rstop_%s' % image
    target_sstop = 'saccades_view_sstop_%s' % image
    target_random = 'saccades_view_random_%s' % image
    
    for i, sample_id in enumerate(do_samples):
        
        logger.info('Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id))
        
        if not db.has_sample(sample_id):
            raise Exception('Sample %s not found in db.' % sample_id)
        
        if not db.has_saccades(sample_id):
            raise Exception('Sample %s does not have saccades table.' % sample_id)
        
        if not db.has_attr(sample_id, 'stimulus_xml'):
            raise Exception('Sample %s does not have the stimulus'
                            ' information ("stimulus_xml")' % sample_id)
       
        # todo: check stale dependencies
        if db.has_table(sample_id, target_start) and \
            db.has_table(sample_id, target_stop) and \
            db.has_table(sample_id, target_rstop) and \
            db.has_table(sample_id, target_sstop) and \
            db.has_table(sample_id, target_random) and \
            not options.nocache:
            logger.info('Targets already computed for %s; skipping' % sample_id)
            continue
        
        # Get the stimulus description
        stimulus_xml = db.get_attr(sample_id, 'stimulus_xml')
        saccades = db.get_saccades(sample_id)
        
        view_start, view_stop, view_rstop, view_random, view_sstop = \
            render_saccades_view(
                saccades=saccades,
                stimulus_xml=stimulus_xml,
                host=options.host,
                white=options.white)
   
        db.set_table(sample_id, target_start, view_start)
        db.set_table(sample_id, target_stop, view_stop)
        db.set_table(sample_id, target_rstop, view_rstop)
        db.set_table(sample_id, target_random, view_random)
        db.set_table(sample_id, target_sstop, view_sstop)
        
        db.release_table(saccades)
        
    db.close()
def main():
    parser = OptionParser(usage=description)
    parser.add_option("--saccade_data", help="Main data directory",
                      default='saccade_data')
    parser.add_option("--db", help="FlydraDB directory")
     
    parser.add_option("--verbose", help='Verbose output',
                      default=False, action="store_true")
        
    (options, args) = parser.parse_args() #@UnusedVariable
    
    if not options.db:
        raise Exception('Please define FlydraDB directory using `--db`.')
    
    def printv(s):
        if options.verbose:
            print(s)
        
    flydra_db = FlydraDB(options.db, create=True)
    
    matlab_dir = options.saccade_data
    for group in os.listdir(matlab_dir):
        group_dir = os.path.join(matlab_dir, group)
        if not os.path.isdir(group_dir):                
            continue
        
        printv("Opening {0}".format(group))
        
#        
#            
#            exp_data, attributes = read_raw_data(filename)
#            
#            consider_importing_processed(flydra_db, sample, exp_data, attributes)
#            
#            flydra_db.set_attr(sample, 'species', attributes['species'])
#            flydra_db.set_attr(sample, 'background', attributes['background'])
#            
#            flydra_db.set_table(sample, EXP_DATA_TABLE, exp_data)
#            flydra_db.add_sample_to_group(sample, group)
#            flydra_db.add_sample_to_group(sample, 'ros')
#            
    
        processed_dir = os.path.join(group_dir, 'processed')
        
        if not os.path.exists(processed_dir):
            printv("No processed data found for group %r." % group)
            continue
        
        for conf in os.listdir(processed_dir):
            # first look for saccades.mat
            saccades_file = os.path.join(processed_dir, conf, 'saccades.mat')
            if os.path.exists(saccades_file):
                printv('Loading from file %r.' % saccades_file)
                saccades = saccades_read_mat(saccades_file)
                samples = numpy.unique(saccades['sample'])
                for sample in samples:
                    if not flydra_db.has_sample(sample):
                        flydra_db.add_sample(sample)
                    flydra_db.add_sample_to_group(sample, group)
                    sample_saccades = saccades[saccades[:]['sample'] == sample]
                    flydra_db.set_table(sample=sample, table=SACCADES_TABLE,
                                        version=conf, data=sample_saccades)
#            else:
#                prefix = 'data_'
#        suffix = '.mat'
#        for file in [file for file in os.listdir(group_dir) 
#            if (file.startswith(prefix)) and file.endswith(suffix)]:
#            
#            sample = file[len(prefix):file.index('.')]
#            
#            if verbose:
#                print("  - Considering sample {0}".format(sample.__repr__()))
#        
#            if not flydra_db.has_sample(sample):
#                flydra_db.add_sample(sample)
#            
#            filename = os.path.join(group_dir, file)
# 
# 
#            
#        else:
#            for conf in os.listdir(processed_dir):                
#                saccades = os.path.join(processed_dir, conf, 'saccades.mat')
#                if os.path.exists(saccades): 
#                    group_record.configurations[conf] = saccades
#                    # add to general list
#                    self.configurations.add(conf)
##                    else:
##                        conf_dir = os.path.join(processed_dir, conf)
##                        for file in [file for file in os.listdir(conf_dir) 
##                            if file.startswith('processed_data_') and file.endswith('.mat')]: 
##                                  id = file[5:-7]
#
#            # if we don't have exp data, get list of samples from
#            # processed data
#            if group_record.configurations and \
#                not group_record.has_experimental_data:
#                saccades = saccades_read_mat(saccades)
#                group_record.samples = set(numpy.unique(saccades['sample']))
#                for sample in group_record.samples:
#                    self.sample2group[sample] = group
#
#        if len(group_record.samples)> 0:
#            self.groups[group] = group_record
#                
#            print "has it", group, group_record.has_experimental_data
#        
    flydra_db.close()
Exemplo n.º 10
0
def main():
    
    parser = OptionParser()
    
    parser.add_option("--db", default='flydra_db', help="FlydraDB directory")

    parser.add_option("--nocache", help="Ignores already computed results.",
                      default=False, action="store_true")

    parser.add_option("--compute_mu", help="Computes mu and optic flow.",
                      default=False, action="store_true")
    
    parser.add_option("--white", help="Computes luminance_w, with the arena"
                      " painted white.", default=False, action="store_true")
    
    parser.add_option("--host", help="Use a remote rfsee. Otherwise, use local process.",
                       default=None)
    
    (options, args) = parser.parse_args()
     
        
    db = FlydraDB(options.db, False)
    
    if args:
        do_samples = args
    else:
        # look for samples with the rows table
        do_samples = db.list_samples()
        do_samples = filter(lambda x: db.has_rows(x), do_samples)
    
    if options.white:
        target = 'luminance_w'
    else:
        target = 'luminance'
    
    for i, sample_id in enumerate(do_samples):
        
        print 'Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id)
        
        if not db.has_sample(sample_id):
            raise Exception('Sample %r not found in db.' % sample_id)
        
        if not db.has_rows(sample_id):
            raise Exception('Sample %r does not have rows table.' % sample_id)
       
        if not db.has_attr(sample_id, 'stimulus_xml'):
            raise Exception('Sample %r does not have the "stimulus_xml" attribute.'
                            %sample_id)
       
        if options.compute_mu:
            if db.has_table(sample_id, 'nearness') and not options.nocache:
                logger.info('Already computed nearness for %r; skipping' % sample_id)
                continue
        else:
            if db.has_table(sample_id, target) and not options.nocache:
                logger.info('Already computed luminance for %r; skipping' % sample_id)
                continue
        
        rows = db.get_rows(sample_id)
        
        stimulus_xml = db.get_attr(sample_id, 'stimulus_xml')
        
        results = render(rows, stimulus_xml, host=options.host,
                         compute_mu=options.compute_mu, white=options.white)
   
        db.set_table(sample_id, target, results['luminance'])
        
        if options.compute_mu:
            db.set_table(sample_id, 'nearness', results['nearness'])
            db.set_table(sample_id, 'retinal_velocities',
                         results['retinal_velocities'])
        
        db.release_table(rows)    
   
    db.close()
Exemplo n.º 11
0
def main_filter(args):
    parser = LenientOptionParser()
    
    parser.add_option("--db", default='flydra_db', help="FlydraDB directory")

    parser.add_option("--min_frames_per_track", default=400,
        help="Minimum number of frames per track [= %default]")

    parser.add_option("--confirm_problems",
                      help="Stop interactively on problems with log files'\
                      '(e.g.: cannot find valid obj_ids) [default: %default]",
                      default=False, action="store_true")

    parser.add_option("--dynamic_model_name",
                      help="Smoothing dynamical model [default: %default]",
                      default="mamarama, units: mm")
    
    parser.add_option("--debug_output", help="Creates debug figures.",
                      default=False, action="store_true")

    parser.add_option("--nocache", help="Ignores already computed results.",
                      default=False, action="store_true")

    parser.add_option("--smoothing", help="Uses Kalman-smoothed data.",
                      default=False, action="store_true")

    
    (options, args) = parser.parse_args(args)
    
    table_name = 'rows' # TODO: use constant
    table_version = "smooth" if options.smoothing else "kf"
    
    
    if not args:
        raise UserError('No files or directories specified.')
         
    if not os.path.exists(options.db):
        os.makedirs(options.db)

    db = FlydraDB(options.db)

    good_files = get_good_files(where=args, pattern="*.kh5",
                                confirm_problems=options.confirm_problems)

    if len(good_files) == 0:
        logger.error("No good files to process")
        sys.exit(1)

    n = len(good_files)
    for i in range(n):
        (filename, obj_ids, stim_fname) = good_files[i]
        
        logger.info('Sample %s/%s: %s' % (i + 1, n, filename))
        
        
        # only maintain basename
        stim = os.path.splitext(os.path.basename(stim_fname))[0]
        sample_id = os.path.splitext(os.path.basename(filename))[0]
        
            
        logger.info("File %d/%d %s %s %s " % 
                    (i, n, str(filename), str(obj_ids), stim_fname))
       
        if (db.has_sample(sample_id) 
            and db.has_table(sample_id, table_name, table_version)
            and not options.nocache):
            logger.info('Sample %r already computed; skipping.'
                        ' (use --nocache to ignore)' % sample_id)
            continue
  
        all_data = [] 

        for obj_id, rows in get_good_smoothed_tracks(#@UnusedVariable
                filename=filename,
                obj_ids=obj_ids,
                min_frames_per_track=options.min_frames_per_track,
                dynamic_model_name=options.dynamic_model_name,
                use_smoothing=options.smoothing):

            filtered = filter_rows(rows, options)
            all_data.append(filtered)
             
        if not all_data:
            logger.info('Not enough data found for %r; skipping.' % sample_id)
            continue

  
        if not db.has_sample(sample_id):
            db.add_sample(sample_id)
        db.set_attr(sample_id, 'stim_fname', stim_fname)
        db.set_attr(sample_id, 'stimulus', stim)
        stim_xml = open(stim_fname).read()
        db.set_attr(sample_id, 'stimulus_xml', stim_xml)
        
        geometry = get_posts_info(stim_xml)
        print(geometry)
        db.set_attr(sample_id, 'posts', geometry['posts'])
        if 'arena' in geometry:
            db.set_attr(sample_id, 'arena', geometry['arena'])


        db.add_sample_to_group(sample_id, stim)
        if stim != 'nopost':
            db.add_sample_to_group(sample_id, 'posts')
            

        rows = numpy.concatenate(all_data)
        db.set_table(sample=sample_id,
                     table=table_name,
                     data=rows,
                     version=table_version)
    
        
        db.set_attr(sample_id, 'filter_time', datetime.now().strftime("%Y%m%d_%H%M%S"))
        db.set_attr(sample_id, 'filter_host', platform.node())
        db.set_attr(sample_id, 'filter_user', get_user())
        db.set_attr(sample_id, 'filter_python_version', platform.python_version())
        db.set_attr(sample_id, 'filter_numpy_version', numpy.version.version)
        
         
    db.close()