def main(): parser = OptionParser(usage=description) parser.add_option("--out", help="Output data directory", default="flydra2ros") parser.add_option("--db", help="Location of input Flydra db.") (options, args) = parser.parse_args() # @UnusedVariable if not options.db: raise Exception("Please define FlydraDB directory using `--db`.") verbose = True def printv(s): if verbose: print(s) db = FlydraDB(options.db, create=False) configuration = "use_for_report" for sample in db.list_samples(): if not db.has_table(sample, table=SACCADES_TABLE, version=configuration): continue group = guess_group(db, sample) magno = {} table = db.get_table(sample, SACCADES_TABLE, configuration) species = db.get_attr(sample, "species", "Dmelanogaster") magno["species"] = species magno["sample"] = sample # _name if db.has_table(sample, EXP_DATA_TABLE): exp_data = db.get_table(sample, EXP_DATA_TABLE) print(exp_data.dtype) timestamp = exp_data[:]["timestamp"] else: timestamp = None magno["use_for_report"] = convert_saccades_to_ros(table, timestamp) db.release_table(table) if timestamp is not None: db.release_table(exp_data) output_dir = os.path.join(options.out, group) filename = os.path.join(output_dir, "magno_%s.mat" % sample) printv("writing to %s" % filename) if not os.path.exists(output_dir): os.makedirs(output_dir) scipy.io.savemat(filename, {"magno": magno}, oned_as="row") # put species and sample print("closing") db.close()
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="FlydraDB directory") parser.add_option("--nocache", help="Ignores already computed results.", default=False, action="store_true") parser.add_option("--sigma", help="Kernel spread (degrees)", type="float", default=6) parser.add_option("--source", default='luminance', help="Source table") parser.add_option("--target", default='contrast', help="Destination table") (options, args) = parser.parse_args() kernel = get_contrast_kernel(sigma_deg=options.sigma, eyes_interact=False) kernel = kernel.astype('float32').copy('C') db = FlydraDB(options.db, False) if args: do_samples = args else: do_samples = db.list_samples() do_samples = filter(lambda x: db.has_table(x, options.source), do_samples) if not do_samples: raise Exception('No samples with table "%s" found. ' % options.source) for i, sample_id in enumerate(do_samples): logger.info('Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id)) if not db.has_sample(sample_id): raise Exception('Sample %s not found in db.' % sample_id) if not db.has_table(sample_id, options.source): raise Exception('Sample %s does not have table %s; skipping.' \ % (sample_id, options.source)) if db.has_table(sample_id, options.target) and not options.nocache: logger.info('Already computed "%s" for %s; skipping' % \ (options.target, sample_id)) continue luminance = db.get_table(sample_id, options.source) contrast = compute_contrast_for_table(luminance, kernel) db.set_table(sample_id, options.target, contrast) db.release_table(luminance) db.close()
def create_and_write_report(flydra_db, sample, image_name): view_start = 'saccades_view_start_%s' % image_name view_stop = 'saccades_view_stop_%s' % image_name view_rstop = 'saccades_view_rstop_%s' % image_name db = FlydraDB(flydra_db, False) saccades = db.get_saccades(sample) values_start = db.get_table(sample, view_start) values_stop = db.get_table(sample, view_stop) values_rstop = db.get_table(sample, view_rstop) r = Report(sample) for i in range(len(saccades)): ri = r.node("saccade-%04d" % i) ri.data_rgb('start', plot_luminance(values_start[i]['value'])) ri.data_rgb('stop', plot_luminance(values_stop[i]['value'])) ri.data_rgb('rstop', plot_luminance(values_rstop[i]['value'])) f = ri.figure(shape=(1, 3)) f.sub('start', 'At saccade start') f.sub('stop', 'At saccade stop') #f.sub('rstop', '(random stop)') db.release_table(saccades) db.release_table(values_start) db.release_table(values_stop) db.release_table(values_rstop) filename = "%s/out/saccade_view_show/%s_%s.html" % (flydra_db, image_name, sample) print "Writing to %s" % filename r.to_html(filename)
def main(): parser = OptionParser() parser.add_option("--db", help="FlydraDB directory") (options, args) = parser.parse_args() #@UnusedVariable if not options.db: raise Exception('Please define the FlydraDB directory using `--db`.') db = FlydraDB(options.db) choose = { 'andrea_detector': { 'Dpseudoobscura': 'threshold16', 'Dananassae': 'threshold10', 'Dhydei': 'threshold7', 'Dmelanogaster': 'threshold1', 'Darizonae': 'threshold9', 'Dmojavensis': 'threshold7', 'peter': 'peters_conf', 'mamaramaposts': 'use_for_report', 'mamaramanoposts': 'use_for_report' }, 'ros_detector': { 'Dpseudoobscura': 'filt_kalman-amp_th_10-th_4', 'Dananassae': 'filt_kalman-amp_th_10-th_4', 'Dhydei': 'filt_kalman-amp_th_10-th_4', 'Dmelanogaster': 'filt_kalman-amp_th_10-th_4', 'Darizonae': 'filt_kalman-amp_th_10-th_4', 'Dmojavensis': 'filt_kalman-amp_th_10-th_4' } } for official, choices in choose.items(): for group, version in choices.items(): print("Group %r: %r -> %r" % (group, version, official)) samples = db.list_samples_for_group(group) for sample in samples: print(" sample %r" % sample) table = db.get_table(sample, SACCADES_TABLE, version=version) copy = numpy.array(table, dtype=table.dtype) db.set_table(sample=sample, table=SACCADES_TABLE, version=official, data=copy) db.release_table(table) db.close()
class FlydraImage(Generator): ''' This block outputs the retinal images from a FlydraDB for a particular sample. ''' Block.alias('flydra_db_image') Block.config('db', 'FlydraDB database directory') Block.config('sample', 'Sample ID -- such as "DATA20080611_191809".') Block.config('image', 'Which retinal image to display.') Block.output('obj_id') Block.output('frame') Block.output('image') def init(self): self.db = FlydraDB(self.config.db) if not self.db.has_sample(self.config.sample): raise ValueError('Sample "%s" not found.' % self.config.sample) if not self.db.has_table(self.config.sample, self.config.image): raise ValueError('Table "%s" not found for sample %s.' % \ (self.config.image, self.config.sample)) self.data = self.db.get_table(self.config.sample, self.config.image) self.next_index = 0 if len(self.data) == 0: self.info('Empty rows for sample %s.' % self.config.sample) self.next_index = None def update(self): row = self.data[self.next_index] t = row['time'] for field in ['obj_id', 'frame']: self.set_output(field, value=row[field], timestamp=t) self.set_output('image', row['value'], timestamp=t) self.next_index += 1 if self.next_index == len(self.data): self.next_index = None def next_data_status(self): # TODO: put new interface if self.next_index is None: # EOF return (False, None) else: return (True, self.data[self.next_index]['time'])
def compute_environment_autocorrelation(db, samples, image, maxlag=50): nsensors = 1398 results = numpy.ndarray(shape=(nsensors, 2 * maxlag + 1)) db = FlydraDB(db, create=False) block_size = 50 num_blocks = int(numpy.ceil(nsensors * 1.0 / block_size)) for b in range(num_blocks): start = block_size * b stop = min(start + block_size, nsensors) progress('Computing autocorrelation', (b, num_blocks)) data = [[] for i in range(nsensors)] for k, sample in enumerate(samples): progress('getting data', (k, len(samples)), sample) table = db.get_table(sample, image) chunk = (table[:]['value'][:, start:stop]).copy() for j, i in enumerate(range(start, stop)): data[i].append(chunk[:, j]) db.release_table(table) for j, i in enumerate(range(start, stop)): progress('Computing correlation', (j, stop - start)) x = numpy.concatenate(data[i]) corr, lags = xcorr(x, maxlag=maxlag) assert(len(lags) == 2 * maxlag + 1) results[i, :] = corr db.close() data = { 'results': results, 'lags': lags } return data
def compute_mean_generic(db, samples, image, operator): ''' db: FlydraDB directory samples: list of IDs ''' db = FlydraDB(db, False) results = { 'samples': {} } ex = Expectation() for i, id in enumerate(samples): progress('Computing mean %s' % image, (i, len(samples)), "Sample %s" % id) if not (db.has_sample(id) and db.has_table(id, image)): raise ValueError('No table "%s" for id %s' % (image, id)) data = db.get_table(id, image) values = data[:]['value'] this = operator(values) # print "id: %s len: %d %d" % (id, len(data), len(values)) ex.update(this, len(data)) results['samples'][id] = this db.release_table(data) results['all'] = ex.get_value() db.close() return results
class SamplesDB: def __init__(self, data, verbose=False): ''' data: base directory ''' if not os.path.exists(data) or not os.path.isdir(data): raise Exception('Could not open directory %s' % data) self.data = data # self.use_cache = True self.use_cache = False self.use_flydra_db = True if self.use_cache: self.open_shelve() if self.use_flydra_db: self.open_flydra_db() self.groups = {} #self.group2samples = {} # maps id to .mat file self.sample2expmat = {} # maps id to .pickle file self.sample2exppickle = {} # list of all configurations self.configurations = set() # maps sample -> group self.sample2group = {} #print "Loading data in %s" % data for group in os.listdir(data): group_dir = os.path.join(data, group) if not os.path.isdir(group_dir): continue # print "Reading group %s" % group group_record = Group() for file in [file for file in os.listdir(group_dir) if file.startswith('data_') and file.endswith('.mat')]: id = file[5:-4] group_record.samples.add(id) self.sample2expmat[id] = os.path.join(group_dir,file) self.sample2group[id] = group for file in [file for file in os.listdir(group_dir) if file.startswith('data_') and file.endswith('.pickle')]: id = file[5:-7] group_record.samples.add(id) self.sample2exppickle[id] = os.path.join(group_dir,file) self.sample2group[id] = group group_record.has_experimental_data = len(group_record.samples) > 0 processed_dir = os.path.join(group_dir, 'processed') if not os.path.exists(processed_dir): if verbose: print "No processed data found for %s." % group pass else: for conf in os.listdir(processed_dir): saccades = os.path.join(processed_dir, conf, 'saccades.mat') if os.path.exists(saccades): group_record.configurations[conf] = saccades # add to general list self.configurations.add(conf) # else: # conf_dir = os.path.join(processed_dir, conf) # for file in [file for file in os.listdir(conf_dir) # if file.startswith('processed_data_') and file.endswith('.mat')]: # id = file[5:-7] # if we don't have exp data, get list of samples from # processed data if group_record.configurations and \ not group_record.has_experimental_data: saccades = saccades_read_mat(saccades) group_record.samples = set(numpy.unique(saccades['sample'])) for sample in group_record.samples: self.sample2group[sample] = group if len(group_record.samples)> 0: self.groups[group] = group_record print "has it", group, group_record.has_experimental_data def open_shelve(self): shelve_fname = os.path.join(self.data, 'shelve') self.shelve = shelve.open(shelve_fname, protocol=pickle.HIGHEST_PROTOCOL) def open_flydra_db(self): self.flydra_db = FlydraDB(os.path.join(self.data, 'sac_flydra_db')) def list_groups(self): """ Returns a list of the groups. """ return natsorted(list(self.groups.keys())) def list_all_samples(self): """ Returns a list of all samples for all groups. """ return natsorted(list(self.sample2group.keys())) def list_samples(self, group): """ Lists the samples in the given group. """ return natsorted(list(self.groups[group].samples)) def list_all_configurations(self): """ Lists all the configurations present in the data. """ return natsorted(self.configurations) def list_configurations(self, group): """ Lists the configurations for the given group. """ return natsorted(list(self.groups[group].configurations.keys())) def get_group_for_sample(self, sample): """ Returns the sample associated to the group. """ return self.sample2group[sample] def get_saccades_for_group(self, group, configuration): """ Returns the saccades for the given group and configuration. If configuration is not passed, we use the default. """ if self.use_flydra_db: table = 'groupsaccades_%s' % configuration if self.flydra_db.has_sample(group) and \ self.flydra_db.has_table(group, table): t = self.flydra_db.get_table(group, table) #value = t.copy() value = t #self.flydra_db.release_table(t) return value if self.use_cache: key = str(('get_saccades_for_group', group, configuration)) if key in self.shelve: return self.shelve[key] filename = self.groups[group].configurations[configuration] saccades = saccades_read_mat(filename) if self.use_flydra_db: if not self.flydra_db.has_sample(group): self.flydra_db.add_sample(group) self.flydra_db.set_table(group, table, saccades) if self.use_cache: self.shelve[key] = saccades return saccades def group_has_experimental_data(self, group): """ Returns true if this group has the raw orientation data. (mamarama has only saccades data. ) """ return self.groups[group].has_experimental_data def has_experimental_data(self, sample): """ Returns true if this sample has the raw orientation data. """ return sample in self.sample2expmat or sample in self. sample2exppickle def get_saccades_for_sample(self, sample, configuration): """ Returns the saccades for the given group and configuration. If configuration is not passed, we use the default. """ if self.use_flydra_db: table = 'saccades_%s' % configuration if self.flydra_db.has_sample(sample) and \ self.flydra_db.has_table(sample, table): t = self.flydra_db.get_table(sample, table) #value = t.copy() value = t #self.flydra_db.release_table(t) return value if self.use_cache: key = str(('get_saccades_for_sample', sample, configuration)) if key in self.shelve: return self.shelve[key] group = self.get_group_for_sample(sample) group_saccades = self.get_saccades_for_group(group, configuration) print group_saccades[0].dtype # with open('tmp.pickle','w') as f: # pickle.dump(f, group_saccades) mine = group_saccades[:]['sample'] == sample saccades = group_saccades[mine] if len(saccades) == 0: raise Exception('No saccades found for %s' % sample) if self.use_flydra_db: if not self.flydra_db.has_sample(sample): self.flydra_db.add_sample(sample) self.flydra_db.set_table(sample, table, saccades) if self.use_cache: self.shelve[key] = saccades return saccades def get_experimental_data(self, sample): # if self.use_flydra_db: # table = 'tethered_data' # if self.flydra_db.has_sample(sample) and \ # self.flydra_db.has_table(sample, table): # t = self.flydra_db.get_table(sample, table) # #value = t.copy() # value = t # #self.flydra_db.release_table(t) # return value # if self.use_cache: if sample in self.shelve: return self.shelve[sample] if sample in self.sample2expmat: data = scipy.io.loadmat(self.sample2expmat[sample], squeeze_me=True) data = data['data'] # convert from array to hash assert isinstance(data, numpy.ndarray) data = dict(map(lambda field: (field, data[field]), data.dtype.fields)) # convert from array to string for k in list(data.keys()): if data[k].dtype.char == 'U': data[k] = str(data[k]) # make sure everything is 1d array def as1d(x): if x.dtype == 'object': x = x.tolist() return x.reshape(len(x)) data['exp_orientation'] = as1d(data['exp_orientation']) data['exp_timestamps'] = as1d(data['exp_timestamps']) elif sample in self.sample2exppickle: with open(self.sample2exppickle[sample], 'rb') as f: data = cPickle.load(f) else: raise Exception('no data for sample %s found' % sample) # if self.use_flydra_db: # if not self.flydra_db.has_sample(sample): # self.flydra_db.add_sample(sample) # # self.flydra_db.set_table(sample, table, data) if self.use_cache: self.shelve[sample] = data return data def __getstate__(self): # do not pickle the shelve all = dict(self.__dict__) all['shelve'] = None all['flydra_db'] = None return all