def main():
parser = OptionParser(usage=description)
parser.add_option("--out", help="Output data directory", default="flydra2ros")
parser.add_option("--db", help="Location of input Flydra db.")
(options, args) = parser.parse_args() # @UnusedVariable
if not options.db:
raise Exception("Please define FlydraDB directory using `--db`.")
verbose = True
def printv(s):
if verbose:
print(s)
db = FlydraDB(options.db, create=False)
configuration = "use_for_report"
for sample in db.list_samples():
if not db.has_table(sample, table=SACCADES_TABLE, version=configuration):
continue
group = guess_group(db, sample)
magno = {}
table = db.get_table(sample, SACCADES_TABLE, configuration)
species = db.get_attr(sample, "species", "Dmelanogaster")
magno["species"] = species
magno["sample"] = sample # _name
if db.has_table(sample, EXP_DATA_TABLE):
exp_data = db.get_table(sample, EXP_DATA_TABLE)
print(exp_data.dtype)
timestamp = exp_data[:]["timestamp"]
else:
timestamp = None
magno["use_for_report"] = convert_saccades_to_ros(table, timestamp)
db.release_table(table)
if timestamp is not None:
db.release_table(exp_data)
output_dir = os.path.join(options.out, group)
filename = os.path.join(output_dir, "magno_%s.mat" % sample)
printv("writing to %s" % filename)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
scipy.io.savemat(filename, {"magno": magno}, oned_as="row")
# put species and sample
print("closing")
db.close()
def main():
parser = OptionParser()
parser.add_option("--db", default='flydra_db', help="FlydraDB directory")
parser.add_option("--nocache", help="Ignores already computed results.",
default=False, action="store_true")
parser.add_option("--sigma", help="Kernel spread (degrees)",
type="float", default=6)
parser.add_option("--source", default='luminance', help="Source table")
parser.add_option("--target", default='contrast', help="Destination table")
(options, args) = parser.parse_args()
kernel = get_contrast_kernel(sigma_deg=options.sigma, eyes_interact=False)
kernel = kernel.astype('float32').copy('C')
db = FlydraDB(options.db, False)
if args:
do_samples = args
else:
do_samples = db.list_samples()
do_samples = filter(lambda x: db.has_table(x, options.source),
do_samples)
if not do_samples:
raise Exception('No samples with table "%s" found. ' % options.source)
for i, sample_id in enumerate(do_samples):
logger.info('Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id))
if not db.has_sample(sample_id):
raise Exception('Sample %s not found in db.' % sample_id)
if not db.has_table(sample_id, options.source):
raise Exception('Sample %s does not have table %s; skipping.' \
% (sample_id, options.source))
if db.has_table(sample_id, options.target) and not options.nocache:
logger.info('Already computed "%s" for %s; skipping' % \
(options.target, sample_id))
continue
luminance = db.get_table(sample_id, options.source)
contrast = compute_contrast_for_table(luminance, kernel)
db.set_table(sample_id, options.target, contrast)
db.release_table(luminance)
db.close()
def create_and_write_report(flydra_db, sample, image_name):
view_start = 'saccades_view_start_%s' % image_name
view_stop = 'saccades_view_stop_%s' % image_name
view_rstop = 'saccades_view_rstop_%s' % image_name
db = FlydraDB(flydra_db, False)
saccades = db.get_saccades(sample)
values_start = db.get_table(sample, view_start)
values_stop = db.get_table(sample, view_stop)
values_rstop = db.get_table(sample, view_rstop)
r = Report(sample)
for i in range(len(saccades)):
ri = r.node("saccade-%04d" % i)
ri.data_rgb('start', plot_luminance(values_start[i]['value']))
ri.data_rgb('stop', plot_luminance(values_stop[i]['value']))
ri.data_rgb('rstop', plot_luminance(values_rstop[i]['value']))
f = ri.figure(shape=(1, 3))
f.sub('start', 'At saccade start')
f.sub('stop', 'At saccade stop')
#f.sub('rstop', '(random stop)')
db.release_table(saccades)
db.release_table(values_start)
db.release_table(values_stop)
db.release_table(values_rstop)
filename = "%s/out/saccade_view_show/%s_%s.html" % (flydra_db, image_name, sample)
print "Writing to %s" % filename
r.to_html(filename)
def main():
parser = OptionParser()
parser.add_option("--db", help="FlydraDB directory")
(options, args) = parser.parse_args() #@UnusedVariable
if not options.db:
raise Exception('Please define the FlydraDB directory using `--db`.')
db = FlydraDB(options.db)
choose = {
'andrea_detector': {
'Dpseudoobscura': 'threshold16',
'Dananassae': 'threshold10',
'Dhydei': 'threshold7',
'Dmelanogaster': 'threshold1',
'Darizonae': 'threshold9',
'Dmojavensis': 'threshold7',
'peter': 'peters_conf',
'mamaramaposts': 'use_for_report',
'mamaramanoposts': 'use_for_report'
},
'ros_detector': {
'Dpseudoobscura': 'filt_kalman-amp_th_10-th_4',
'Dananassae': 'filt_kalman-amp_th_10-th_4',
'Dhydei': 'filt_kalman-amp_th_10-th_4',
'Dmelanogaster': 'filt_kalman-amp_th_10-th_4',
'Darizonae': 'filt_kalman-amp_th_10-th_4',
'Dmojavensis': 'filt_kalman-amp_th_10-th_4'
}
}
for official, choices in choose.items():
for group, version in choices.items():
print("Group %r: %r -> %r" % (group, version, official))
samples = db.list_samples_for_group(group)
for sample in samples:
print(" sample %r" % sample)
table = db.get_table(sample, SACCADES_TABLE, version=version)
copy = numpy.array(table, dtype=table.dtype)
db.set_table(sample=sample, table=SACCADES_TABLE,
version=official, data=copy)
db.release_table(table)
db.close()
class FlydraImage(Generator):
''' This block outputs the retinal images from a FlydraDB for
a particular sample. '''
Block.alias('flydra_db_image')
Block.config('db', 'FlydraDB database directory')
Block.config('sample', 'Sample ID -- such as "DATA20080611_191809".')
Block.config('image', 'Which retinal image to display.')
Block.output('obj_id')
Block.output('frame')
Block.output('image')
def init(self):
self.db = FlydraDB(self.config.db)
if not self.db.has_sample(self.config.sample):
raise ValueError('Sample "%s" not found.' % self.config.sample)
if not self.db.has_table(self.config.sample, self.config.image):
raise ValueError('Table "%s" not found for sample %s.' % \
(self.config.image, self.config.sample))
self.data = self.db.get_table(self.config.sample, self.config.image)
self.next_index = 0
if len(self.data) == 0:
self.info('Empty rows for sample %s.' % self.config.sample)
self.next_index = None
def update(self):
row = self.data[self.next_index]
t = row['time']
for field in ['obj_id', 'frame']:
self.set_output(field, value=row[field], timestamp=t)
self.set_output('image', row['value'], timestamp=t)
self.next_index += 1
if self.next_index == len(self.data):
self.next_index = None
def next_data_status(self):
# TODO: put new interface
if self.next_index is None: # EOF
return (False, None)
else:
return (True, self.data[self.next_index]['time'])
def compute_environment_autocorrelation(db, samples, image, maxlag=50):
nsensors = 1398
results = numpy.ndarray(shape=(nsensors, 2 * maxlag + 1))
db = FlydraDB(db, create=False)
block_size = 50
num_blocks = int(numpy.ceil(nsensors * 1.0 / block_size))
for b in range(num_blocks):
start = block_size * b
stop = min(start + block_size, nsensors)
progress('Computing autocorrelation', (b, num_blocks))
data = [[] for i in range(nsensors)]
for k, sample in enumerate(samples):
progress('getting data', (k, len(samples)), sample)
table = db.get_table(sample, image)
chunk = (table[:]['value'][:, start:stop]).copy()
for j, i in enumerate(range(start, stop)):
data[i].append(chunk[:, j])
db.release_table(table)
for j, i in enumerate(range(start, stop)):
progress('Computing correlation', (j, stop - start))
x = numpy.concatenate(data[i])
corr, lags = xcorr(x, maxlag=maxlag)
assert(len(lags) == 2 * maxlag + 1)
results[i, :] = corr
db.close()
data = {
'results': results,
'lags': lags
}
return data
def compute_mean_generic(db, samples, image, operator):
'''
db: FlydraDB directory
samples: list of IDs
'''
db = FlydraDB(db, False)
results = { 'samples': {} }
ex = Expectation()
for i, id in enumerate(samples):
progress('Computing mean %s' % image,
(i, len(samples)), "Sample %s" % id)
if not (db.has_sample(id) and db.has_table(id, image)):
raise ValueError('No table "%s" for id %s' % (image, id))
data = db.get_table(id, image)
values = data[:]['value']
this = operator(values)
# print "id: %s len: %d %d" % (id, len(data), len(values))
ex.update(this, len(data))
results['samples'][id] = this
db.release_table(data)
results['all'] = ex.get_value()
db.close()
return results
class SamplesDB:
def __init__(self, data, verbose=False):
''' data: base directory '''
if not os.path.exists(data) or not os.path.isdir(data):
raise Exception('Could not open directory %s' % data)
self.data = data
# self.use_cache = True
self.use_cache = False
self.use_flydra_db = True
if self.use_cache:
self.open_shelve()
if self.use_flydra_db:
self.open_flydra_db()
self.groups = {}
#self.group2samples = {}
# maps id to .mat file
self.sample2expmat = {}
# maps id to .pickle file
self.sample2exppickle = {}
# list of all configurations
self.configurations = set()
# maps sample -> group
self.sample2group = {}
#print "Loading data in %s" % data
for group in os.listdir(data):
group_dir = os.path.join(data, group)
if not os.path.isdir(group_dir):
continue
# print "Reading group %s" % group
group_record = Group()
for file in [file for file in os.listdir(group_dir)
if file.startswith('data_') and file.endswith('.mat')]:
id = file[5:-4]
group_record.samples.add(id)
self.sample2expmat[id] = os.path.join(group_dir,file)
self.sample2group[id] = group
for file in [file for file in os.listdir(group_dir)
if file.startswith('data_') and file.endswith('.pickle')]:
id = file[5:-7]
group_record.samples.add(id)
self.sample2exppickle[id] = os.path.join(group_dir,file)
self.sample2group[id] = group
group_record.has_experimental_data = len(group_record.samples) > 0
processed_dir = os.path.join(group_dir, 'processed')
if not os.path.exists(processed_dir):
if verbose:
print "No processed data found for %s." % group
pass
else:
for conf in os.listdir(processed_dir):
saccades = os.path.join(processed_dir, conf, 'saccades.mat')
if os.path.exists(saccades):
group_record.configurations[conf] = saccades
# add to general list
self.configurations.add(conf)
# else:
# conf_dir = os.path.join(processed_dir, conf)
# for file in [file for file in os.listdir(conf_dir)
# if file.startswith('processed_data_') and file.endswith('.mat')]:
# id = file[5:-7]
# if we don't have exp data, get list of samples from
# processed data
if group_record.configurations and \
not group_record.has_experimental_data:
saccades = saccades_read_mat(saccades)
group_record.samples = set(numpy.unique(saccades['sample']))
for sample in group_record.samples:
self.sample2group[sample] = group
if len(group_record.samples)> 0:
self.groups[group] = group_record
print "has it", group, group_record.has_experimental_data
def open_shelve(self):
shelve_fname = os.path.join(self.data, 'shelve')
self.shelve = shelve.open(shelve_fname, protocol=pickle.HIGHEST_PROTOCOL)
def open_flydra_db(self):
self.flydra_db = FlydraDB(os.path.join(self.data, 'sac_flydra_db'))
def list_groups(self):
""" Returns a list of the groups. """
return natsorted(list(self.groups.keys()))
def list_all_samples(self):
""" Returns a list of all samples for all groups. """
return natsorted(list(self.sample2group.keys()))
def list_samples(self, group):
""" Lists the samples in the given group. """
return natsorted(list(self.groups[group].samples))
def list_all_configurations(self):
""" Lists all the configurations present in the data. """
return natsorted(self.configurations)
def list_configurations(self, group):
""" Lists the configurations for the given group. """
return natsorted(list(self.groups[group].configurations.keys()))
def get_group_for_sample(self, sample):
""" Returns the sample associated to the group. """
return self.sample2group[sample]
def get_saccades_for_group(self, group, configuration):
""" Returns the saccades for the given group and configuration.
If configuration is not passed, we use the default.
"""
if self.use_flydra_db:
table = 'groupsaccades_%s' % configuration
if self.flydra_db.has_sample(group) and \
self.flydra_db.has_table(group, table):
t = self.flydra_db.get_table(group, table)
#value = t.copy()
value = t
#self.flydra_db.release_table(t)
return value
if self.use_cache:
key = str(('get_saccades_for_group', group, configuration))
if key in self.shelve:
return self.shelve[key]
filename = self.groups[group].configurations[configuration]
saccades = saccades_read_mat(filename)
if self.use_flydra_db:
if not self.flydra_db.has_sample(group):
self.flydra_db.add_sample(group)
self.flydra_db.set_table(group, table, saccades)
if self.use_cache:
self.shelve[key] = saccades
return saccades
def group_has_experimental_data(self, group):
""" Returns true if this group has the raw orientation data.
(mamarama has only saccades data. ) """
return self.groups[group].has_experimental_data
def has_experimental_data(self, sample):
""" Returns true if this sample has the raw orientation data. """
return sample in self.sample2expmat or sample in self. sample2exppickle
def get_saccades_for_sample(self, sample, configuration):
""" Returns the saccades for the given group and configuration.
If configuration is not passed, we use the default.
"""
if self.use_flydra_db:
table = 'saccades_%s' % configuration
if self.flydra_db.has_sample(sample) and \
self.flydra_db.has_table(sample, table):
t = self.flydra_db.get_table(sample, table)
#value = t.copy()
value = t
#self.flydra_db.release_table(t)
return value
if self.use_cache:
key = str(('get_saccades_for_sample', sample, configuration))
if key in self.shelve:
return self.shelve[key]
group = self.get_group_for_sample(sample)
group_saccades = self.get_saccades_for_group(group, configuration)
print group_saccades[0].dtype
# with open('tmp.pickle','w') as f:
# pickle.dump(f, group_saccades)
mine = group_saccades[:]['sample'] == sample
saccades = group_saccades[mine]
if len(saccades) == 0:
raise Exception('No saccades found for %s' % sample)
if self.use_flydra_db:
if not self.flydra_db.has_sample(sample):
self.flydra_db.add_sample(sample)
self.flydra_db.set_table(sample, table, saccades)
if self.use_cache:
self.shelve[key] = saccades
return saccades
def get_experimental_data(self, sample):
# if self.use_flydra_db:
# table = 'tethered_data'
# if self.flydra_db.has_sample(sample) and \
# self.flydra_db.has_table(sample, table):
# t = self.flydra_db.get_table(sample, table)
# #value = t.copy()
# value = t
# #self.flydra_db.release_table(t)
# return value
#
if self.use_cache:
if sample in self.shelve:
return self.shelve[sample]
if sample in self.sample2expmat:
data = scipy.io.loadmat(self.sample2expmat[sample], squeeze_me=True)
data = data['data']
# convert from array to hash
assert isinstance(data, numpy.ndarray)
data = dict(map(lambda field: (field, data[field]), data.dtype.fields))
# convert from array to string
for k in list(data.keys()):
if data[k].dtype.char == 'U':
data[k] = str(data[k])
# make sure everything is 1d array
def as1d(x):
if x.dtype == 'object':
x = x.tolist()
return x.reshape(len(x))
data['exp_orientation'] = as1d(data['exp_orientation'])
data['exp_timestamps'] = as1d(data['exp_timestamps'])
elif sample in self.sample2exppickle:
with open(self.sample2exppickle[sample], 'rb') as f:
data = cPickle.load(f)
else:
raise Exception('no data for sample %s found' % sample)
# if self.use_flydra_db:
# if not self.flydra_db.has_sample(sample):
# self.flydra_db.add_sample(sample)
#
# self.flydra_db.set_table(sample, table, data)
if self.use_cache:
self.shelve[sample] = data
return data
def __getstate__(self):
# do not pickle the shelve
all = dict(self.__dict__)
all['shelve'] = None
all['flydra_db'] = None
return all
