def get_saccades_for_group(self, group, configuration): """ Returns the saccades for the given group and configuration. If configuration is not passed, we use the default. """ if self.use_flydra_db: table = 'groupsaccades_%s' % configuration if self.flydra_db.has_sample(group) and \ self.flydra_db.has_table(group, table): t = self.flydra_db.get_table(group, table) #value = t.copy() value = t #self.flydra_db.release_table(t) return value if self.use_cache: key = str(('get_saccades_for_group', group, configuration)) if key in self.shelve: return self.shelve[key] filename = self.groups[group].configurations[configuration] saccades = saccades_read_mat(filename) if self.use_flydra_db: if not self.flydra_db.has_sample(group): self.flydra_db.add_sample(group) self.flydra_db.set_table(group, table, saccades) if self.use_cache: self.shelve[key] = saccades return saccades
def __init__(self, data, verbose=False): ''' data: base directory ''' if not os.path.exists(data) or not os.path.isdir(data): raise Exception('Could not open directory %s' % data) self.data = data # self.use_cache = True self.use_cache = False self.use_flydra_db = True if self.use_cache: self.open_shelve() if self.use_flydra_db: self.open_flydra_db() self.groups = {} #self.group2samples = {} # maps id to .mat file self.sample2expmat = {} # maps id to .pickle file self.sample2exppickle = {} # list of all configurations self.configurations = set() # maps sample -> group self.sample2group = {} #print "Loading data in %s" % data for group in os.listdir(data): group_dir = os.path.join(data, group) if not os.path.isdir(group_dir): continue # print "Reading group %s" % group group_record = Group() for file in [file for file in os.listdir(group_dir) if file.startswith('data_') and file.endswith('.mat')]: id = file[5:-4] group_record.samples.add(id) self.sample2expmat[id] = os.path.join(group_dir,file) self.sample2group[id] = group for file in [file for file in os.listdir(group_dir) if file.startswith('data_') and file.endswith('.pickle')]: id = file[5:-7] group_record.samples.add(id) self.sample2exppickle[id] = os.path.join(group_dir,file) self.sample2group[id] = group group_record.has_experimental_data = len(group_record.samples) > 0 processed_dir = os.path.join(group_dir, 'processed') if not os.path.exists(processed_dir): if verbose: print "No processed data found for %s." % group pass else: for conf in os.listdir(processed_dir): saccades = os.path.join(processed_dir, conf, 'saccades.mat') if os.path.exists(saccades): group_record.configurations[conf] = saccades # add to general list self.configurations.add(conf) # else: # conf_dir = os.path.join(processed_dir, conf) # for file in [file for file in os.listdir(conf_dir) # if file.startswith('processed_data_') and file.endswith('.mat')]: # id = file[5:-7] # if we don't have exp data, get list of samples from # processed data if group_record.configurations and \ not group_record.has_experimental_data: saccades = saccades_read_mat(saccades) group_record.samples = set(numpy.unique(saccades['sample'])) for sample in group_record.samples: self.sample2group[sample] = group if len(group_record.samples)> 0: self.groups[group] = group_record print "has it", group, group_record.has_experimental_data
def main(): parser = OptionParser(usage=description) parser.add_option("--saccade_data", help="Main data directory", default='saccade_data') parser.add_option("--db", help="FlydraDB directory") parser.add_option("--verbose", help='Verbose output', default=False, action="store_true") (options, args) = parser.parse_args() #@UnusedVariable if not options.db: raise Exception('Please define FlydraDB directory using `--db`.') def printv(s): if options.verbose: print(s) flydra_db = FlydraDB(options.db, create=True) matlab_dir = options.saccade_data for group in os.listdir(matlab_dir): group_dir = os.path.join(matlab_dir, group) if not os.path.isdir(group_dir): continue printv("Opening {0}".format(group)) # # # exp_data, attributes = read_raw_data(filename) # # consider_importing_processed(flydra_db, sample, exp_data, attributes) # # flydra_db.set_attr(sample, 'species', attributes['species']) # flydra_db.set_attr(sample, 'background', attributes['background']) # # flydra_db.set_table(sample, EXP_DATA_TABLE, exp_data) # flydra_db.add_sample_to_group(sample, group) # flydra_db.add_sample_to_group(sample, 'ros') # processed_dir = os.path.join(group_dir, 'processed') if not os.path.exists(processed_dir): printv("No processed data found for group %r." % group) continue for conf in os.listdir(processed_dir): # first look for saccades.mat saccades_file = os.path.join(processed_dir, conf, 'saccades.mat') if os.path.exists(saccades_file): printv('Loading from file %r.' % saccades_file) saccades = saccades_read_mat(saccades_file) samples = numpy.unique(saccades['sample']) for sample in samples: if not flydra_db.has_sample(sample): flydra_db.add_sample(sample) flydra_db.add_sample_to_group(sample, group) sample_saccades = saccades[saccades[:]['sample'] == sample] flydra_db.set_table(sample=sample, table=SACCADES_TABLE, version=conf, data=sample_saccades) # else: # prefix = 'data_' # suffix = '.mat' # for file in [file for file in os.listdir(group_dir) # if (file.startswith(prefix)) and file.endswith(suffix)]: # # sample = file[len(prefix):file.index('.')] # # if verbose: # print(" - Considering sample {0}".format(sample.__repr__())) # # if not flydra_db.has_sample(sample): # flydra_db.add_sample(sample) # # filename = os.path.join(group_dir, file) # # # # else: # for conf in os.listdir(processed_dir): # saccades = os.path.join(processed_dir, conf, 'saccades.mat') # if os.path.exists(saccades): # group_record.configurations[conf] = saccades # # add to general list # self.configurations.add(conf) ## else: ## conf_dir = os.path.join(processed_dir, conf) ## for file in [file for file in os.listdir(conf_dir) ## if file.startswith('processed_data_') and file.endswith('.mat')]: ## id = file[5:-7] # # # if we don't have exp data, get list of samples from # # processed data # if group_record.configurations and \ # not group_record.has_experimental_data: # saccades = saccades_read_mat(saccades) # group_record.samples = set(numpy.unique(saccades['sample'])) # for sample in group_record.samples: # self.sample2group[sample] = group # # if len(group_record.samples)> 0: # self.groups[group] = group_record # # print "has it", group, group_record.has_experimental_data # flydra_db.close()