def init(self):
if self.config.db is not None and self.config.sample is not None:
db = FlydraDB(self.config.db)
stimulus_xml = db.get_attr(self.config.sample, 'stimulus_xml')
self.arena_info = get_posts_info(stimulus_xml)
else:
self.arena_info = None
class FlydraImage(Generator):
''' This block outputs the retinal images from a FlydraDB for
a particular sample. '''
Block.alias('flydra_db_image')
Block.config('db', 'FlydraDB database directory')
Block.config('sample', 'Sample ID -- such as "DATA20080611_191809".')
Block.config('image', 'Which retinal image to display.')
Block.output('obj_id')
Block.output('frame')
Block.output('image')
def init(self):
self.db = FlydraDB(self.config.db)
if not self.db.has_sample(self.config.sample):
raise ValueError('Sample "%s" not found.' % self.config.sample)
if not self.db.has_table(self.config.sample, self.config.image):
raise ValueError('Table "%s" not found for sample %s.' % \
(self.config.image, self.config.sample))
self.data = self.db.get_table(self.config.sample, self.config.image)
self.next_index = 0
if len(self.data) == 0:
self.info('Empty rows for sample %s.' % self.config.sample)
self.next_index = None
def update(self):
row = self.data[self.next_index]
t = row['time']
for field in ['obj_id', 'frame']:
self.set_output(field, value=row[field], timestamp=t)
self.set_output('image', row['value'], timestamp=t)
self.next_index += 1
if self.next_index == len(self.data):
self.next_index = None
def next_data_status(self):
# TODO: put new interface
if self.next_index is None: # EOF
return (False, None)
else:
return (True, self.data[self.next_index]['time'])
def main():
parser = OptionParser()
parser.add_option("--db", help="FlydraDB directory")
(options, args) = parser.parse_args() #@UnusedVariable
if not options.db:
raise Exception('Please define the FlydraDB directory using `--db`.')
db = FlydraDB(options.db)
choose = {
'andrea_detector': {
'Dpseudoobscura': 'threshold16',
'Dananassae': 'threshold10',
'Dhydei': 'threshold7',
'Dmelanogaster': 'threshold1',
'Darizonae': 'threshold9',
'Dmojavensis': 'threshold7',
'peter': 'peters_conf',
'mamaramaposts': 'use_for_report',
'mamaramanoposts': 'use_for_report'
},
'ros_detector': {
'Dpseudoobscura': 'filt_kalman-amp_th_10-th_4',
'Dananassae': 'filt_kalman-amp_th_10-th_4',
'Dhydei': 'filt_kalman-amp_th_10-th_4',
'Dmelanogaster': 'filt_kalman-amp_th_10-th_4',
'Darizonae': 'filt_kalman-amp_th_10-th_4',
'Dmojavensis': 'filt_kalman-amp_th_10-th_4'
}
}
for official, choices in choose.items():
for group, version in choices.items():
print("Group %r: %r -> %r" % (group, version, official))
samples = db.list_samples_for_group(group)
for sample in samples:
print(" sample %r" % sample)
table = db.get_table(sample, SACCADES_TABLE, version=version)
copy = numpy.array(table, dtype=table.dtype)
db.set_table(sample=sample, table=SACCADES_TABLE,
version=official, data=copy)
db.release_table(table)
db.close()
def main():
parser = OptionParser(usage=description)
parser.add_option("--db", default='flydra_db', help="FlydraDB directory")
parser.add_option("--interactive",
help="Start compmake interactive session."
" Otherwise run in batch mode",
default=False, action="store_true")
(options, args) = parser.parse_args() #@UnusedVariable
db = FlydraDB(options.db, False)
set_namespace('video_contrast')
samples = db.list_samples()
if not samples:
print 'No samples found'
for id in samples:
if db.has_rows(id) and db.has_table(id, 'contrast') and \
db.has_table(id, 'luminance'):
config = {'sample': id, 'db': options.db}
comp(pg, 'flydra_display_contrast', config,
job_id="flydra_display_contrast:%s" % id)
if options.interactive:
# start interactive session
compmake_console()
else:
# batch mode
# try to do everything
batch_command('make all')
# start the console if we are not done
# (that is, make all failed for some reason)
todo = list(parse_job_list('todo'))
if todo:
print('Still %d jobs to do.' % len(todo))
sys.exit(-2)
def main():
set_namespace('env_stats')
parser = OptionParser()
parser.add_option("--db", default='flydra_db', help="FlydraDB directory")
(options, args) = parser.parse_args() #@UnusedVariable
db = FlydraDB(options.db, False)
outdir = os.path.join(options.db, 'out/environment_stats')
images = ["luminance", "contrast", "luminance_w", "contrast_w",
"hluminance_w", "hcontrast_w"]
for image in images:
samples = [x for x in db.list_samples()
if db.get_attr(x, 'stimulus', None) != 'nopost' and
db.has_table(x, image)]
if not samples:
print "No samples for %s" % samples
continue
comp_prefix(image)
data = comp(compute_environment_autocorrelation, options.db, samples, image)
comp(create_report, data, image, outdir)
db.close()
compmake_console()
def main():
parser = OptionParser(usage=description)
parser.add_option("--db", default='flydra_db', help="FlydraDB directory")
(options, args) = parser.parse_args() #@UnusedVariable
if options.db is None:
logger.error('Please specify a directory using --db.')
sys.exit(-1)
db = FlydraDB(options.db)
for sample in db.list_samples():
if not db.has_rows(sample):
continue
rows = db.get_rows(sample)
v = rows[:]['linear_velocity_modulus']
perc = [1, 5, 95, 99]
print "Sample %s" % sample
print " linear_velocity_modulus"
for p in perc:
s = scipy.stats.scoreatpercentile(v, p)
print ' - %2d%% %.3f m/s' % (p, s)
db.release_table(rows)
class FlydraData(Generator):
''' This block outputs the data from a FlydraDB for
a particular sample. '''
Block.alias('flydra_db_source')
Block.config('db', 'Database directory')
Block.config('sample', 'Sample ID -- such as "DATA20080611_191809".')
for f in fields:
Block.output(f)
Block.output('stimulus_xml')
def init(self):
self.db = FlydraDB(self.config.db, False)
self.rows = self.db.get_rows(self.config.sample)
self.next_index = 0
def update(self):
row = self.rows[self.next_index]
t = row['time']
for field in fields:
self.set_output(field, value=row[field].copy(), timestamp=t)
if self.db.has_attr(self.config.sample, 'stimulus_xml'):
stim_xml = self.db.get_attr(self.config.sample, 'stimulus_xml')
self.set_output('stimulus_xml', stim_xml, timestamp=t)
self.next_index += 1
if self.next_index == len(self.rows):
self.next_index = None
def next_data_status(self):
# TODO: put new interface
if self.next_index is None: # EOF
return (False, None)
else:
return (True, self.rows[self.next_index]['time'])
def init(self):
self.db = FlydraDB(self.config.db)
if not self.db.has_sample(self.config.sample):
raise ValueError('Sample "%s" not found.' % self.config.sample)
if not self.db.has_table(self.config.sample, self.config.image):
raise ValueError('Table "%s" not found for sample %s.' % \
(self.config.image, self.config.sample))
self.data = self.db.get_table(self.config.sample, self.config.image)
self.next_index = 0
if len(self.data) == 0:
self.info('Empty rows for sample %s.' % self.config.sample)
self.next_index = None
def main():
parser = OptionParser()
parser.add_option("--db", default='flydra_db', help="Data directory")
parser.add_option("--image", default="luminance",
help="Rendered image to use -- "
" corresponding to image 'saccades_view_{start,stop}_X'")
parser.add_option("--interactive",
help="Start an interactive compmake session."
" Otherwise run in batch mode. ",
default=False, action="store_true")
(options, args) = parser.parse_args() #@UnusedVariable
if options.db is None:
logger.error('Please specify a directory using --db.')
sys.exit(-1)
view_start = 'saccades_view_start_%s' % options.image
view_stop = 'saccades_view_stop_%s' % options.image
view_rstop = 'saccades_view_rstop_%s' % options.image
db = FlydraDB(options.db, False)
# all samples with enough data
all_available = lambda x: db.has_saccades(x) and \
db.has_table(x, view_start) and \
db.has_table(x, view_stop) and \
db.has_table(x, view_rstop)
samples = filter(all_available, db.list_samples())
set_namespace('saccade_view_show_%s' % options.image)
for sample in samples:
comp_prefix(sample)
comp(create_and_write_report, options.db, sample, options.image)
if options.interactive:
# start interactive session
compmake_console()
else:
# batch mode
# try to do everything
batch_command('make all')
# start the console if we are not done
# (that is, make all failed for some reason)
todo = list(parse_job_list('todo'))
if todo:
logger.info('Still %d jobs to do.' % len(todo))
sys.exit(-2)
def get_all_data_for_signal(db, samples, interval_function,
signal, signal_component):
db = FlydraDB(db, False)
all = []
for id in samples:
if not db.has_rows(id):
logger.warning('Could not find rows table for %s; skipping.' %
(id))
continue
rows_table = db.get_rows(id)
try:
interval = interval_function(db, id, rows_table)
except Exception as e:
logger.warning('Cannot compute interval for sample %s: %s '\
% (id, e))
db.release_table(rows_table)
continue
rows = rows_table[interval]
s = extract_signal(rows, signal, signal_component)
all.append(s)
db.release_table(rows_table)
db.close()
return numpy.concatenate(all)
def compute_mean_generic(db, samples, image, operator):
'''
db: FlydraDB directory
samples: list of IDs
'''
db = FlydraDB(db, False)
results = { 'samples': {} }
ex = Expectation()
for i, id in enumerate(samples):
progress('Computing mean %s' % image,
(i, len(samples)), "Sample %s" % id)
if not (db.has_sample(id) and db.has_table(id, image)):
raise ValueError('No table "%s" for id %s' % (image, id))
data = db.get_table(id, image)
values = data[:]['value']
this = operator(values)
# print "id: %s len: %d %d" % (id, len(data), len(values))
ex.update(this, len(data))
results['samples'][id] = this
db.release_table(data)
results['all'] = ex.get_value()
db.close()
return results
def compute_environment_autocorrelation(db, samples, image, maxlag=50):
nsensors = 1398
results = numpy.ndarray(shape=(nsensors, 2 * maxlag + 1))
db = FlydraDB(db, create=False)
block_size = 50
num_blocks = int(numpy.ceil(nsensors * 1.0 / block_size))
for b in range(num_blocks):
start = block_size * b
stop = min(start + block_size, nsensors)
progress('Computing autocorrelation', (b, num_blocks))
data = [[] for i in range(nsensors)]
for k, sample in enumerate(samples):
progress('getting data', (k, len(samples)), sample)
table = db.get_table(sample, image)
chunk = (table[:]['value'][:, start:stop]).copy()
for j, i in enumerate(range(start, stop)):
data[i].append(chunk[:, j])
db.release_table(table)
for j, i in enumerate(range(start, stop)):
progress('Computing correlation', (j, stop - start))
x = numpy.concatenate(data[i])
corr, lags = xcorr(x, maxlag=maxlag)
assert(len(lags) == 2 * maxlag + 1)
results[i, :] = corr
db.close()
data = {
'results': results,
'lags': lags
}
return data
def main():
parser = OptionParser(usage=description)
parser.add_option("--out", help="Output data directory", default="flydra2ros")
parser.add_option("--db", help="Location of input Flydra db.")
(options, args) = parser.parse_args() # @UnusedVariable
if not options.db:
raise Exception("Please define FlydraDB directory using `--db`.")
verbose = True
def printv(s):
if verbose:
print(s)
db = FlydraDB(options.db, create=False)
configuration = "use_for_report"
for sample in db.list_samples():
if not db.has_table(sample, table=SACCADES_TABLE, version=configuration):
continue
group = guess_group(db, sample)
magno = {}
table = db.get_table(sample, SACCADES_TABLE, configuration)
species = db.get_attr(sample, "species", "Dmelanogaster")
magno["species"] = species
magno["sample"] = sample # _name
if db.has_table(sample, EXP_DATA_TABLE):
exp_data = db.get_table(sample, EXP_DATA_TABLE)
print(exp_data.dtype)
timestamp = exp_data[:]["timestamp"]
else:
timestamp = None
magno["use_for_report"] = convert_saccades_to_ros(table, timestamp)
db.release_table(table)
if timestamp is not None:
db.release_table(exp_data)
output_dir = os.path.join(options.out, group)
filename = os.path.join(output_dir, "magno_%s.mat" % sample)
printv("writing to %s" % filename)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
scipy.io.savemat(filename, {"magno": magno}, oned_as="row")
# put species and sample
print("closing")
db.close()
def main():
parser = OptionParser(usage=description)
parser.add_option("--saccade_data", help="Main data directory",
default='saccade_data')
parser.add_option("--db", help="FlydraDB directory")
parser.add_option("--verbose", help='Verbose output',
default=False, action="store_true")
(options, args) = parser.parse_args() #@UnusedVariable
if not options.db:
raise Exception('Please define FlydraDB directory using `--db`.')
def printv(s):
if options.verbose:
print(s)
flydra_db = FlydraDB(options.db, create=True)
matlab_dir = options.saccade_data
for group in os.listdir(matlab_dir):
group_dir = os.path.join(matlab_dir, group)
if not os.path.isdir(group_dir):
continue
printv("Opening {0}".format(group))
#
#
# exp_data, attributes = read_raw_data(filename)
#
# consider_importing_processed(flydra_db, sample, exp_data, attributes)
#
# flydra_db.set_attr(sample, 'species', attributes['species'])
# flydra_db.set_attr(sample, 'background', attributes['background'])
#
# flydra_db.set_table(sample, EXP_DATA_TABLE, exp_data)
# flydra_db.add_sample_to_group(sample, group)
# flydra_db.add_sample_to_group(sample, 'ros')
#
processed_dir = os.path.join(group_dir, 'processed')
if not os.path.exists(processed_dir):
printv("No processed data found for group %r." % group)
continue
for conf in os.listdir(processed_dir):
# first look for saccades.mat
saccades_file = os.path.join(processed_dir, conf, 'saccades.mat')
if os.path.exists(saccades_file):
printv('Loading from file %r.' % saccades_file)
saccades = saccades_read_mat(saccades_file)
samples = numpy.unique(saccades['sample'])
for sample in samples:
if not flydra_db.has_sample(sample):
flydra_db.add_sample(sample)
flydra_db.add_sample_to_group(sample, group)
sample_saccades = saccades[saccades[:]['sample'] == sample]
flydra_db.set_table(sample=sample, table=SACCADES_TABLE,
version=conf, data=sample_saccades)
# else:
# prefix = 'data_'
# suffix = '.mat'
# for file in [file for file in os.listdir(group_dir)
# if (file.startswith(prefix)) and file.endswith(suffix)]:
#
# sample = file[len(prefix):file.index('.')]
#
# if verbose:
# print(" - Considering sample {0}".format(sample.__repr__()))
#
# if not flydra_db.has_sample(sample):
# flydra_db.add_sample(sample)
#
# filename = os.path.join(group_dir, file)
#
#
#
# else:
# for conf in os.listdir(processed_dir):
# saccades = os.path.join(processed_dir, conf, 'saccades.mat')
# if os.path.exists(saccades):
# group_record.configurations[conf] = saccades
# # add to general list
# self.configurations.add(conf)
## else:
## conf_dir = os.path.join(processed_dir, conf)
## for file in [file for file in os.listdir(conf_dir)
## if file.startswith('processed_data_') and file.endswith('.mat')]:
## id = file[5:-7]
#
# # if we don't have exp data, get list of samples from
# # processed data
# if group_record.configurations and \
# not group_record.has_experimental_data:
# saccades = saccades_read_mat(saccades)
# group_record.samples = set(numpy.unique(saccades['sample']))
# for sample in group_record.samples:
# self.sample2group[sample] = group
#
# if len(group_record.samples)> 0:
# self.groups[group] = group_record
#
# print "has it", group, group_record.has_experimental_data
#
flydra_db.close()
def init(self):
self.db = FlydraDB(self.config.db, False)
self.rows = self.db.get_rows(self.config.sample)
self.next_index = 0
def main():
parser = OptionParser(usage=description)
parser.add_option("--db", default='flydra_db', help="FlydraDB directory")
parser.add_option("--nocache", help="Ignores already computed results.",
default=False, action="store_true")
parser.add_option("--white", help="Computes luminance_w, with the arena"
" painted white.", default=False, action="store_true")
parser.add_option("--host", help="Use a remote rfsee. Otherwise,"
"use local process.", default=None)
(options, args) = parser.parse_args()
if options.db is None:
logger.error('Please specify a directory using --db.')
sys.exit(-1)
db = FlydraDB(options.db)
if args:
do_samples = args
else:
# look for samples with the rows table
all_samples = db.list_samples()
do_samples = filter(lambda x: db.has_saccades(x) and
db.has_attr(x, 'stimulus_xml'),
all_samples)
logger.info('Found %d/%d samples with saccades and stimulus info.' %
(len(do_samples), len(all_samples)))
image = 'luminance_w' if options.white else 'luminance'
target_start = 'saccades_view_start_%s' % image
target_stop = 'saccades_view_stop_%s' % image
target_rstop = 'saccades_view_rstop_%s' % image
target_sstop = 'saccades_view_sstop_%s' % image
target_random = 'saccades_view_random_%s' % image
for i, sample_id in enumerate(do_samples):
logger.info('Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id))
if not db.has_sample(sample_id):
raise Exception('Sample %s not found in db.' % sample_id)
if not db.has_saccades(sample_id):
raise Exception('Sample %s does not have saccades table.' % sample_id)
if not db.has_attr(sample_id, 'stimulus_xml'):
raise Exception('Sample %s does not have the stimulus'
' information ("stimulus_xml")' % sample_id)
# todo: check stale dependencies
if db.has_table(sample_id, target_start) and \
db.has_table(sample_id, target_stop) and \
db.has_table(sample_id, target_rstop) and \
db.has_table(sample_id, target_sstop) and \
db.has_table(sample_id, target_random) and \
not options.nocache:
logger.info('Targets already computed for %s; skipping' % sample_id)
continue
# Get the stimulus description
stimulus_xml = db.get_attr(sample_id, 'stimulus_xml')
saccades = db.get_saccades(sample_id)
view_start, view_stop, view_rstop, view_random, view_sstop = \
render_saccades_view(
saccades=saccades,
stimulus_xml=stimulus_xml,
host=options.host,
white=options.white)
db.set_table(sample_id, target_start, view_start)
db.set_table(sample_id, target_stop, view_stop)
db.set_table(sample_id, target_rstop, view_rstop)
db.set_table(sample_id, target_random, view_random)
db.set_table(sample_id, target_sstop, view_sstop)
db.release_table(saccades)
db.close()
def main():
parser = OptionParser(usage=description)
parser.add_option("--saccade_data", help="Main data directory",
default='saccade_data')
parser.add_option("--db", help='Location of output Flydra db.')
(options, args) = parser.parse_args() #@UnusedVariable
if not options.db:
raise Exception('Please define FlydraDB directory using `--db`.')
verbose = True
flydra_db = FlydraDB(options.db, create=True)
matlab_dir = options.saccade_data
for group in os.listdir(matlab_dir):
group_dir = os.path.join(matlab_dir, group)
if not os.path.isdir(group_dir):
continue
if verbose:
print("Opening {0}".format(group))
for file in [file for file in os.listdir(group_dir)
if (file.startswith('magno_')) \
and file.endswith('.mat')]:
sample = file[file.index('_') + 1:file.index('.')]
if verbose:
print(" - Considering sample {0}".format(sample.__repr__()))
if not flydra_db.has_sample(sample):
flydra_db.add_sample(sample)
flydra_db.add_sample_to_group(sample, group)
# flydra_db.add_sample_to_group(sample, 'ros')
filename = os.path.join(group_dir, file)
exp_data, attributes = read_raw_data(filename)
consider_importing_processed(flydra_db, sample, exp_data, attributes)
flydra_db.set_attr(sample, 'species', attributes['species'])
flydra_db.set_attr(sample, 'background', attributes['background'])
flydra_db.set_table(sample, EXP_DATA_TABLE, exp_data)
flydra_db.close()
def main():
parser = OptionParser()
parser.add_option("--db", default='flydra_db', help="FlydraDB directory")
parser.add_option("--nocache", help="Ignores already computed results.",
default=False, action="store_true")
parser.add_option("--compute_mu", help="Computes mu and optic flow.",
default=False, action="store_true")
parser.add_option("--white", help="Computes luminance_w, with the arena"
" painted white.", default=False, action="store_true")
parser.add_option("--host", help="Use a remote rfsee. Otherwise, use local process.",
default=None)
(options, args) = parser.parse_args() #@UnusedVariable
db = FlydraDB(options.db, False)
# look for samples with the rows table
do_samples = db.list_samples()
do_samples = filter(lambda x: db.has_rows(x) and
db.get_attr(x, 'stimulus') == 'nopost',
do_samples)
if not do_samples:
raise Exception('Cannot find samples to hallucinate about.')
print "Summary, including nopost."
for s in sorted(get_db_stimulus_stats(db, include_nopost=True),
key=(lambda x:-x.total_length)):
print "stimulus: {s.stimulus:>10} samples: {s.total_number:>5} "\
" total length: {len:>5} minutes".format(s=s, len=s.total_length / (60 * 60))
stimulus_to_use = list(get_stimulus_to_use(db, len(do_samples)))
for i, sample in enumerate(do_samples):
stimulus = stimulus_to_use[i][0]
print sample, stimulus
if options.white:
target = 'hluminance_w'
else:
target = 'hluminance'
for i, sample_id in enumerate(do_samples):
stimulus = stimulus_to_use[i][0]
stimulus_xml = stimulus_to_use[i][1]
print 'Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id)
if not db.has_sample(sample_id):
raise Exception('Sample %s not found in db.' % sample_id)
if not db.has_rows(sample_id):
raise Exception('Sample %s does not have rows table.' % sample_id)
if options.compute_mu:
if db.has_table(sample_id, 'nearness') and not options.nocache:
logger.info('Already computed nearness for %s; skipping' % sample_id)
continue
else:
if db.has_table(sample_id, target) and not options.nocache:
logger.info('Already computed luminance for %s; skipping' % sample_id)
continue
rows = db.get_rows(sample_id)
results = render(rows, stimulus_xml, host=options.host,
compute_mu=options.compute_mu, white=options.white)
db.set_table(sample_id, target, results['luminance'])
if options.compute_mu:
db.set_table(sample_id, 'hnearness', results['nearness'])
db.set_table(sample_id, 'hretinal_velocities',
results['retinal_velocities'])
db.release_table(rows)
db.close()
def main():
parser = OptionParser()
parser.add_option("--db", default='flydra_db', help="FlydraDB directory")
parser.add_option("--nocache", help="Ignores already computed results.",
default=False, action="store_true")
parser.add_option("--compute_mu", help="Computes mu and optic flow.",
default=False, action="store_true")
parser.add_option("--white", help="Computes luminance_w, with the arena"
" painted white.", default=False, action="store_true")
parser.add_option("--host", help="Use a remote rfsee. Otherwise, use local process.",
default=None)
(options, args) = parser.parse_args()
db = FlydraDB(options.db, False)
if args:
do_samples = args
else:
# look for samples with the rows table
do_samples = db.list_samples()
do_samples = filter(lambda x: db.has_rows(x), do_samples)
if options.white:
target = 'luminance_w'
else:
target = 'luminance'
for i, sample_id in enumerate(do_samples):
print 'Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id)
if not db.has_sample(sample_id):
raise Exception('Sample %r not found in db.' % sample_id)
if not db.has_rows(sample_id):
raise Exception('Sample %r does not have rows table.' % sample_id)
if not db.has_attr(sample_id, 'stimulus_xml'):
raise Exception('Sample %r does not have the "stimulus_xml" attribute.'
%sample_id)
if options.compute_mu:
if db.has_table(sample_id, 'nearness') and not options.nocache:
logger.info('Already computed nearness for %r; skipping' % sample_id)
continue
else:
if db.has_table(sample_id, target) and not options.nocache:
logger.info('Already computed luminance for %r; skipping' % sample_id)
continue
rows = db.get_rows(sample_id)
stimulus_xml = db.get_attr(sample_id, 'stimulus_xml')
results = render(rows, stimulus_xml, host=options.host,
compute_mu=options.compute_mu, white=options.white)
db.set_table(sample_id, target, results['luminance'])
if options.compute_mu:
db.set_table(sample_id, 'nearness', results['nearness'])
db.set_table(sample_id, 'retinal_velocities',
results['retinal_velocities'])
db.release_table(rows)
db.close()
def open_flydra_db(self):
self.flydra_db = FlydraDB(os.path.join(self.data, 'sac_flydra_db'))
def main():
parser = OptionParser()
parser.add_option("--db", default='flydra_db', help="FlydraDB directory")
parser.add_option("--nocache", help="Ignores already computed results.",
default=False, action="store_true")
parser.add_option("--sigma", help="Kernel spread (degrees)",
type="float", default=6)
parser.add_option("--source", default='luminance', help="Source table")
parser.add_option("--target", default='contrast', help="Destination table")
(options, args) = parser.parse_args()
kernel = get_contrast_kernel(sigma_deg=options.sigma, eyes_interact=False)
kernel = kernel.astype('float32').copy('C')
db = FlydraDB(options.db, False)
if args:
do_samples = args
else:
do_samples = db.list_samples()
do_samples = filter(lambda x: db.has_table(x, options.source),
do_samples)
if not do_samples:
raise Exception('No samples with table "%s" found. ' % options.source)
for i, sample_id in enumerate(do_samples):
logger.info('Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id))
if not db.has_sample(sample_id):
raise Exception('Sample %s not found in db.' % sample_id)
if not db.has_table(sample_id, options.source):
raise Exception('Sample %s does not have table %s; skipping.' \
% (sample_id, options.source))
if db.has_table(sample_id, options.target) and not options.nocache:
logger.info('Already computed "%s" for %s; skipping' % \
(options.target, sample_id))
continue
luminance = db.get_table(sample_id, options.source)
contrast = compute_contrast_for_table(luminance, kernel)
db.set_table(sample_id, options.target, contrast)
db.release_table(luminance)
db.close()
class SamplesDB:
def __init__(self, data, verbose=False):
''' data: base directory '''
if not os.path.exists(data) or not os.path.isdir(data):
raise Exception('Could not open directory %s' % data)
self.data = data
# self.use_cache = True
self.use_cache = False
self.use_flydra_db = True
if self.use_cache:
self.open_shelve()
if self.use_flydra_db:
self.open_flydra_db()
self.groups = {}
#self.group2samples = {}
# maps id to .mat file
self.sample2expmat = {}
# maps id to .pickle file
self.sample2exppickle = {}
# list of all configurations
self.configurations = set()
# maps sample -> group
self.sample2group = {}
#print "Loading data in %s" % data
for group in os.listdir(data):
group_dir = os.path.join(data, group)
if not os.path.isdir(group_dir):
continue
# print "Reading group %s" % group
group_record = Group()
for file in [file for file in os.listdir(group_dir)
if file.startswith('data_') and file.endswith('.mat')]:
id = file[5:-4]
group_record.samples.add(id)
self.sample2expmat[id] = os.path.join(group_dir,file)
self.sample2group[id] = group
for file in [file for file in os.listdir(group_dir)
if file.startswith('data_') and file.endswith('.pickle')]:
id = file[5:-7]
group_record.samples.add(id)
self.sample2exppickle[id] = os.path.join(group_dir,file)
self.sample2group[id] = group
group_record.has_experimental_data = len(group_record.samples) > 0
processed_dir = os.path.join(group_dir, 'processed')
if not os.path.exists(processed_dir):
if verbose:
print "No processed data found for %s." % group
pass
else:
for conf in os.listdir(processed_dir):
saccades = os.path.join(processed_dir, conf, 'saccades.mat')
if os.path.exists(saccades):
group_record.configurations[conf] = saccades
# add to general list
self.configurations.add(conf)
# else:
# conf_dir = os.path.join(processed_dir, conf)
# for file in [file for file in os.listdir(conf_dir)
# if file.startswith('processed_data_') and file.endswith('.mat')]:
# id = file[5:-7]
# if we don't have exp data, get list of samples from
# processed data
if group_record.configurations and \
not group_record.has_experimental_data:
saccades = saccades_read_mat(saccades)
group_record.samples = set(numpy.unique(saccades['sample']))
for sample in group_record.samples:
self.sample2group[sample] = group
if len(group_record.samples)> 0:
self.groups[group] = group_record
print "has it", group, group_record.has_experimental_data
def open_shelve(self):
shelve_fname = os.path.join(self.data, 'shelve')
self.shelve = shelve.open(shelve_fname, protocol=pickle.HIGHEST_PROTOCOL)
def open_flydra_db(self):
self.flydra_db = FlydraDB(os.path.join(self.data, 'sac_flydra_db'))
def list_groups(self):
""" Returns a list of the groups. """
return natsorted(list(self.groups.keys()))
def list_all_samples(self):
""" Returns a list of all samples for all groups. """
return natsorted(list(self.sample2group.keys()))
def list_samples(self, group):
""" Lists the samples in the given group. """
return natsorted(list(self.groups[group].samples))
def list_all_configurations(self):
""" Lists all the configurations present in the data. """
return natsorted(self.configurations)
def list_configurations(self, group):
""" Lists the configurations for the given group. """
return natsorted(list(self.groups[group].configurations.keys()))
def get_group_for_sample(self, sample):
""" Returns the sample associated to the group. """
return self.sample2group[sample]
def get_saccades_for_group(self, group, configuration):
""" Returns the saccades for the given group and configuration.
If configuration is not passed, we use the default.
"""
if self.use_flydra_db:
table = 'groupsaccades_%s' % configuration
if self.flydra_db.has_sample(group) and \
self.flydra_db.has_table(group, table):
t = self.flydra_db.get_table(group, table)
#value = t.copy()
value = t
#self.flydra_db.release_table(t)
return value
if self.use_cache:
key = str(('get_saccades_for_group', group, configuration))
if key in self.shelve:
return self.shelve[key]
filename = self.groups[group].configurations[configuration]
saccades = saccades_read_mat(filename)
if self.use_flydra_db:
if not self.flydra_db.has_sample(group):
self.flydra_db.add_sample(group)
self.flydra_db.set_table(group, table, saccades)
if self.use_cache:
self.shelve[key] = saccades
return saccades
def group_has_experimental_data(self, group):
""" Returns true if this group has the raw orientation data.
(mamarama has only saccades data. ) """
return self.groups[group].has_experimental_data
def has_experimental_data(self, sample):
""" Returns true if this sample has the raw orientation data. """
return sample in self.sample2expmat or sample in self. sample2exppickle
def get_saccades_for_sample(self, sample, configuration):
""" Returns the saccades for the given group and configuration.
If configuration is not passed, we use the default.
"""
if self.use_flydra_db:
table = 'saccades_%s' % configuration
if self.flydra_db.has_sample(sample) and \
self.flydra_db.has_table(sample, table):
t = self.flydra_db.get_table(sample, table)
#value = t.copy()
value = t
#self.flydra_db.release_table(t)
return value
if self.use_cache:
key = str(('get_saccades_for_sample', sample, configuration))
if key in self.shelve:
return self.shelve[key]
group = self.get_group_for_sample(sample)
group_saccades = self.get_saccades_for_group(group, configuration)
print group_saccades[0].dtype
# with open('tmp.pickle','w') as f:
# pickle.dump(f, group_saccades)
mine = group_saccades[:]['sample'] == sample
saccades = group_saccades[mine]
if len(saccades) == 0:
raise Exception('No saccades found for %s' % sample)
if self.use_flydra_db:
if not self.flydra_db.has_sample(sample):
self.flydra_db.add_sample(sample)
self.flydra_db.set_table(sample, table, saccades)
if self.use_cache:
self.shelve[key] = saccades
return saccades
def get_experimental_data(self, sample):
# if self.use_flydra_db:
# table = 'tethered_data'
# if self.flydra_db.has_sample(sample) and \
# self.flydra_db.has_table(sample, table):
# t = self.flydra_db.get_table(sample, table)
# #value = t.copy()
# value = t
# #self.flydra_db.release_table(t)
# return value
#
if self.use_cache:
if sample in self.shelve:
return self.shelve[sample]
if sample in self.sample2expmat:
data = scipy.io.loadmat(self.sample2expmat[sample], squeeze_me=True)
data = data['data']
# convert from array to hash
assert isinstance(data, numpy.ndarray)
data = dict(map(lambda field: (field, data[field]), data.dtype.fields))
# convert from array to string
for k in list(data.keys()):
if data[k].dtype.char == 'U':
data[k] = str(data[k])
# make sure everything is 1d array
def as1d(x):
if x.dtype == 'object':
x = x.tolist()
return x.reshape(len(x))
data['exp_orientation'] = as1d(data['exp_orientation'])
data['exp_timestamps'] = as1d(data['exp_timestamps'])
elif sample in self.sample2exppickle:
with open(self.sample2exppickle[sample], 'rb') as f:
data = cPickle.load(f)
else:
raise Exception('no data for sample %s found' % sample)
# if self.use_flydra_db:
# if not self.flydra_db.has_sample(sample):
# self.flydra_db.add_sample(sample)
#
# self.flydra_db.set_table(sample, table, data)
if self.use_cache:
self.shelve[sample] = data
return data
def __getstate__(self):
# do not pickle the shelve
all = dict(self.__dict__)
all['shelve'] = None
all['flydra_db'] = None
return all
def main():
parser = OptionParser(usage=description)
parser.add_option("--flydra_db", default="saccade_data_flydradb", help="Main data directory")
parser.add_option(
"--interactive", action="store_true", default=False, help="Starts an interactive compmake session."
)
parser.add_option("--report", default="saccade_report", help="Saccade report directory")
parser.add_option("--groups", default=None, help="Which groups to consider")
parser.add_option("--configurations", default=None, help="Which configurations to consider")
parser.add_option("--combid", default=None, help="How to name this combination of groups/configs.")
(options, args) = parser.parse_args() # @UnusedVariable
if args:
raise Exception("Spurious arguments %r." % args)
db = FlydraDB(options.flydra_db)
robust_split = lambda s: filter(lambda x: x, s.split(","))
if not options.groups in [None, "all"]:
groups = robust_split(options.groups)
if not groups:
raise Exception("No groups specified.")
groupset = "_".join(groups)
else:
groups = db.list_groups()
groupset = "all"
if not groups:
raise Exception("No groups found.")
if not options.configurations in [None, "all"]:
configurations = robust_split(options.configurations)
if not configurations:
raise Exception("No configuration specified")
confset = "_".join(configurations)
else:
configurations = db.list_all_versions_for_table(SACCADES_TABLE)
confset = "all"
configurations = set()
for group in groups:
configurations.update(db.list_versions_for_table_in_group(group, SACCADES_TABLE))
configurations = natsorted(configurations)
if not configurations:
raise Exception("No valid versions of table %r found." % SACCADES_TABLE)
print ("I will consider the configurations: %r" % configurations)
if options.combid is None:
combination = "%s_%s" % (groupset, confset)
else:
combination = options.combid
print ("I call this combination %r." % combination)
output_dir = os.path.join(options.report, combination)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
set_namespace("master_plot_%s" % combination)
# we maintain several indices
# key = (group, configuration, plot)
index_group_plots = {}
# key = (sample, plot)
index_sample_expdata_plots = {}
# key = (sample, configuration, plot)
index_sample_saccades_plots = {}
# First we index the DB
print ("Looking for data in database...")
all_samples = set()
group2samples = {}
configurations_for_group = {}
group_has_exp_data = {}
for group in groups:
group2samples[group] = db.list_samples_for_group(group)
all_samples.update(group2samples[group])
if not group2samples[group]:
raise Exception("Empty group %r." % group)
available = db.list_versions_for_table_in_group(group=group, table=SACCADES_TABLE)
configurations_for_group[group] = set(configurations).intersection(available)
if not configurations_for_group[group]:
print ("No configurations found for %r; available %r" % (group, available))
group_has_exp_data[group] = db.group_has_table(group, EXP_DATA_TABLE)
all_samples = natsorted(all_samples)
# print info
print ("Summary:")
for group in groups:
msg = " group {group:>20} samples: {nsamples:3} " " configurations: {nconf:3} raw data? {data}".format(
group=group,
nsamples=len(group2samples[group]),
nconf=len(configurations_for_group[group]),
data=group_has_exp_data[group],
)
print (msg)
# TODO: iterate by sample, not by group
for group in groups:
for configuration in configurations_for_group[group]:
for plot in group_plots:
job_id = "%s-%s-%s" % (group, configuration, plot.id)
index_group_plots[(group, configuration, plot.id)] = comp(
wrap_group_plot, options.flydra_db, group, configuration, plot.command, plot.args, job_id=job_id
)
for sample, plot in prod(group2samples[group], sample_saccades_plots):
job_id = "%s-%s-%s" % (sample, configuration, plot.id)
key = (sample, configuration, plot.id)
if key in index_sample_saccades_plots:
# we already did it as part of another group
continue
index_sample_saccades_plots[key] = comp(
wrap_sample_saccades_plot,
options.flydra_db,
sample,
configuration,
plot.command,
plot.args,
job_id=job_id,
)
if group_has_exp_data[group]:
for sample, plot in prod(group2samples[group], sample_expdata_plots):
job_id = "%s-%s" % (sample, plot.id)
key = (sample, plot.id)
if key in index_sample_expdata_plots:
# we already did it as part of another group
continue
index_sample_expdata_plots[key] = comp(
wrap_sample_expdata_plot, options.flydra_db, sample, plot.command, plot.args, job_id=job_id
)
# now we create the indices
# fix configuration, function; iterate groups
for configuration, plot in itertools.product(configurations, group_plots):
subs = []
descs = []
page_id = "%s.%s" % (configuration, plot.id)
for group, group_desc in order_groups(groups):
if not configuration in configurations_for_group[group]:
continue
descs.append(group_desc)
subs.append(index_group_plots[(group, configuration, plot.id)])
if not subs:
raise Exception("no groups for configuration %r." % configuration)
job_id = page_id
comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id)
comp(
create_gui,
filename=os.path.join(output_dir, "group_plots.html"),
menus=[
("Detector", configurations, configurations),
("Plot/table", map(lambda x: x.id, group_plots), map(lambda x: x.description, group_plots)),
],
job_id="gui-group_plots",
)
# fix group, function; iterate samples
for group in groups:
if not group_has_exp_data[group]:
continue
for plot in sample_expdata_plots:
subs = []
descs = []
for sample in group2samples[group]:
descs.append(sample)
subs.append(index_sample_expdata_plots[(sample, plot.id)])
page_id = "%s.%s" % (group, plot.id)
job_id = page_id
comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id)
# get the ordered group lists and desc
ordered_groups = map(lambda t: t[0], order_groups(groups))
ordered_groups_desc = map(lambda t: t[1], order_groups(groups))
comp(
create_gui,
filename=os.path.join(output_dir, "expdata_plots.html"),
menus=[
("Group", ordered_groups, ordered_groups_desc),
(
"Plot/table",
map(lambda x: x.id, sample_expdata_plots),
map(lambda x: x.description, sample_expdata_plots),
),
],
job_id="gui-expdata_plots",
)
# fix configuration, group, function; iterate samples
for group in groups:
for configuration in configurations:
if not configuration in configurations_for_group[group]:
for plot in sample_saccades_plots:
page_id = "%s.%s.%s" % (configuration, group, plot.id)
comp(
write_empty,
page_id,
output_dir,
"Group %r has not been processed with algorithm %r." % (group, configuration),
job_id=page_id,
)
continue
for plot in sample_saccades_plots:
subs = []
descs = []
for sample in group2samples[group]:
descs.append(sample)
r = index_sample_saccades_plots[(sample, configuration, plot.id)]
subs.append(r)
page_id = "%s.%s.%s" % (configuration, group, plot.id)
job_id = page_id
comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id)
comp(
create_gui,
filename=os.path.join(output_dir, "saccade_plots.html"),
menus=[
("Detector", configurations, configurations),
("Group", ordered_groups, ordered_groups_desc),
(
"Plot/table",
map(lambda x: x.id, sample_saccades_plots),
map(lambda x: x.description, sample_saccades_plots),
),
],
job_id="gui-saccade_plots",
)
# fix configuration, sample; plot fullsscreen
for group in groups:
for configuration in configurations:
for sample in group2samples[group]:
# XXX make it clenaer
if not configuration in configurations_for_group[group]:
for plot in sample_fullscreen_plots:
page_id = "%s.%s.%s" % (sample, configuration, plot.id)
comp(
write_empty,
page_id,
output_dir,
'Group %s has not been processed with algorithm "%s".' % (group, configuration),
job_id=page_id,
)
# print "skipping sample %s group %s config %s" %\
# (sample,group, configuration)
continue
if not group_has_exp_data[group]:
for plot in sample_fullscreen_plots:
page_id = "%s.%s.%s" % (sample, configuration, plot.id)
comp(
write_empty,
page_id,
output_dir,
"Group %r does not have raw experimental data." % (group),
job_id=page_id,
)
continue
for plot in sample_fullscreen_plots:
job_id = "%s-%s-%s" % (sample, configuration, plot.id)
# FIXME: error if sample in 2 groups
job = comp(
wrap_sample_saccades_plot,
options.flydra_db,
sample,
configuration,
plot.command,
plot.args,
job_id=job_id,
)
page_id = "%s.%s.%s" % (sample, configuration, plot.id)
comp(write_report, job, output_dir, page_id, job_id=job_id + "-write_report")
comp(
create_gui,
filename=os.path.join(output_dir, "sample_fullscreen_plots.html"),
menus=[
("Sample", all_samples, all_samples),
("Detector", configurations, configurations),
(
"Plot/table",
map(lambda x: x.id, sample_fullscreen_plots),
map(lambda x: x.description, sample_fullscreen_plots),
),
],
job_id="gui-sample_fullscreen_plots",
)
tabs = [
(
"group_plots",
"By group",
"This set displays one plot/table for each group of samples. "
"You have the further choice of detection algorithm and plot/table to display.",
),
(
"saccade_plots",
"By sample",
"This set displays one plot/table for each individual sample. "
"You have the further choice of which group to consider, which "
"detection algorithm, and which plot/table to display.",
),
(
"expdata_plots",
"By sample (raw)",
"This set displays one plot/table for each individual sample, "
" produced from the raw data (no saccade detection, so no choice of detector). "
"You have the further choice of which group to consider, "
"and which plot/table to display."
" Note that some samples might be missing; for example, we don't use "
" raw orientation data for the Mamarama samples.",
),
(
"sample_fullscreen_plots",
"By sample, single page",
"This set displays one entire page for each sample. "
"You have the further choice of sample, "
"detection algorithm, and which plot/table to display.",
),
]
comp(create_main_gui, tabs, filename=os.path.join(output_dir, "main.html"), job_id="gui-main")
if options.interactive:
# start interactive session
compmake_console()
else:
# batch mode
# try to do everything
batch_command("make all")
# start the console if we are not done
# (that is, make all failed for some reason)
todo = list(parse_job_list("todo"))
if todo:
print ("Still %d jobs to do." % len(todo))
sys.exit(-2)
def main_filter(args):
parser = LenientOptionParser()
parser.add_option("--db", default='flydra_db', help="FlydraDB directory")
parser.add_option("--min_frames_per_track", default=400,
help="Minimum number of frames per track [= %default]")
parser.add_option("--confirm_problems",
help="Stop interactively on problems with log files'\
'(e.g.: cannot find valid obj_ids) [default: %default]",
default=False, action="store_true")
parser.add_option("--dynamic_model_name",
help="Smoothing dynamical model [default: %default]",
default="mamarama, units: mm")
parser.add_option("--debug_output", help="Creates debug figures.",
default=False, action="store_true")
parser.add_option("--nocache", help="Ignores already computed results.",
default=False, action="store_true")
parser.add_option("--smoothing", help="Uses Kalman-smoothed data.",
default=False, action="store_true")
(options, args) = parser.parse_args(args)
table_name = 'rows' # TODO: use constant
table_version = "smooth" if options.smoothing else "kf"
if not args:
raise UserError('No files or directories specified.')
if not os.path.exists(options.db):
os.makedirs(options.db)
db = FlydraDB(options.db)
good_files = get_good_files(where=args, pattern="*.kh5",
confirm_problems=options.confirm_problems)
if len(good_files) == 0:
logger.error("No good files to process")
sys.exit(1)
n = len(good_files)
for i in range(n):
(filename, obj_ids, stim_fname) = good_files[i]
logger.info('Sample %s/%s: %s' % (i + 1, n, filename))
# only maintain basename
stim = os.path.splitext(os.path.basename(stim_fname))[0]
sample_id = os.path.splitext(os.path.basename(filename))[0]
logger.info("File %d/%d %s %s %s " %
(i, n, str(filename), str(obj_ids), stim_fname))
if (db.has_sample(sample_id)
and db.has_table(sample_id, table_name, table_version)
and not options.nocache):
logger.info('Sample %r already computed; skipping.'
' (use --nocache to ignore)' % sample_id)
continue
all_data = []
for obj_id, rows in get_good_smoothed_tracks(#@UnusedVariable
filename=filename,
obj_ids=obj_ids,
min_frames_per_track=options.min_frames_per_track,
dynamic_model_name=options.dynamic_model_name,
use_smoothing=options.smoothing):
filtered = filter_rows(rows, options)
all_data.append(filtered)
if not all_data:
logger.info('Not enough data found for %r; skipping.' % sample_id)
continue
if not db.has_sample(sample_id):
db.add_sample(sample_id)
db.set_attr(sample_id, 'stim_fname', stim_fname)
db.set_attr(sample_id, 'stimulus', stim)
stim_xml = open(stim_fname).read()
db.set_attr(sample_id, 'stimulus_xml', stim_xml)
geometry = get_posts_info(stim_xml)
print(geometry)
db.set_attr(sample_id, 'posts', geometry['posts'])
if 'arena' in geometry:
db.set_attr(sample_id, 'arena', geometry['arena'])
db.add_sample_to_group(sample_id, stim)
if stim != 'nopost':
db.add_sample_to_group(sample_id, 'posts')
rows = numpy.concatenate(all_data)
db.set_table(sample=sample_id,
table=table_name,
data=rows,
version=table_version)
db.set_attr(sample_id, 'filter_time', datetime.now().strftime("%Y%m%d_%H%M%S"))
db.set_attr(sample_id, 'filter_host', platform.node())
db.set_attr(sample_id, 'filter_user', get_user())
db.set_attr(sample_id, 'filter_python_version', platform.python_version())
db.set_attr(sample_id, 'filter_numpy_version', numpy.version.version)
db.close()
def main():
parser = OptionParser(usage=description)
parser.add_option(
"--db", default='flydra_db_directory', help="FlydraDB directory")
parser.add_option("--model", help="ProcGraph model name.")
parser.add_option(
"--needs",
help="Comma-separated list of tables required",
default="rows,luminance")
parser.add_option(
"--interactive",
help="Start compmake interactive session."
" Otherwise run in batch mode",
default=False,
action="store_true")
(options, args) = parser.parse_args()
if options.model is None:
print "Please specify the model."
sys.exit(-3)
print("Using FlydraDB directory %r." % options.db)
db = FlydraDB(options.db, False)
# TODO: make the storage inside options.db?
set_namespace('run_pg_model_%s' % options.model)
tables = options.needs.split(',')
if args:
samples = args
for sample in samples:
if not db.has_sample(sample):
raise Exception('Unknown sample %r' % sample)
else:
samples = db.list_samples()
if not samples:
print 'No samples found'
num_ok = 0
for id in samples:
enough = all(map(lambda t: db.has_table(id, t), tables))
if not enough:
continue
num_ok += 1
config = {'sample': id, 'db': options.db}
comp(pg, options.model, config, job_id=id)
logger.info(
"Found %d/%d samples with tables %s." % (num_ok, len(samples), tables))
if options.interactive:
# start interactive session
compmake_console()
else:
# batch mode
# try to do everything
batch_command('make all')
# start the console if we are not done
# (that is, make all failed for some reason)
todo = list(parse_job_list('todo'))
if todo:
logger.info('Still %d jobs to do.' % len(todo))
sys.exit(-2)
def main():
parser = OptionParser()
parser.add_option("--db", default='flydra_db', help="Data directory")
parser.add_option("--image", help="Which image to plot.")
parser.add_option("--filter", help="Which procgraph filter to use to plot.",
default="flydra_simple_video_filter")
parser.add_option("--interactive",
help="Start compmake interactive session."
" Otherwise run in batch mode",
default=False, action="store_true")
(options, args) = parser.parse_args()
if options.image is None:
print "Usage: %s [--db DB] --image <image> [ids]" % sys.argv[0]
sys.exit(-1)
db = FlydraDB(options.db)
set_namespace('video_image_%s' % options.image)
if args:
samples = args
else:
# look for samples with the rows table
samples = db.list_samples()
samples = filter(lambda x: db.has_table(x, options.image), samples)
if not samples:
raise Exception('No samples found at all with available image "%s".' % \
options.image)
for id in samples:
if not db.has_table(id, options.image):
raise Exception('Sample %s does not have table "%s".' %
(id, options.image))
config = {'sample': id,
'db': options.db,
'image': options.image,
'filter': options.filter}
comp(pg, 'flydra_simple_video', config,
job_id="%s" % id)
if options.interactive:
# start interactive session
compmake_console()
else:
# batch mode
# try to do everything
batch_command('make all')
# start the console if we are not done
# (that is, make all failed for some reason)
todo = list(parse_job_list('todo'))
if todo:
print('Still %d jobs to do.' % len(todo))
sys.exit(-2)
def create_and_write_report(flydra_db, sample, image_name):
view_start = 'saccades_view_start_%s' % image_name
view_stop = 'saccades_view_stop_%s' % image_name
view_rstop = 'saccades_view_rstop_%s' % image_name
db = FlydraDB(flydra_db, False)
saccades = db.get_saccades(sample)
values_start = db.get_table(sample, view_start)
values_stop = db.get_table(sample, view_stop)
values_rstop = db.get_table(sample, view_rstop)
r = Report(sample)
for i in range(len(saccades)):
ri = r.node("saccade-%04d" % i)
ri.data_rgb('start', plot_luminance(values_start[i]['value']))
ri.data_rgb('stop', plot_luminance(values_stop[i]['value']))
ri.data_rgb('rstop', plot_luminance(values_rstop[i]['value']))
f = ri.figure(shape=(1, 3))
f.sub('start', 'At saccade start')
f.sub('stop', 'At saccade stop')
#f.sub('rstop', '(random stop)')
db.release_table(saccades)
db.release_table(values_start)
db.release_table(values_stop)
db.release_table(values_rstop)
filename = "%s/out/saccade_view_show/%s_%s.html" % (flydra_db, image_name, sample)
print "Writing to %s" % filename
r.to_html(filename)
def main():
parser = OptionParser(usage=description)
parser.add_option("--db", default='flydra_db', help="Data directory")
parser.add_option("--interactive", default=False, action="store_true",
help="Start a compmake interactive session."
" Otherwise run in batch mode")
parser.add_option("--empty_group_ok",
default=False, action="store_true",
help="do not give up if one group does not have samples ")
(options, args) = parser.parse_args() #@UnusedVariable
if options.db is None:
logger.error('Please specify a directory using --db.')
sys.exit(-1)
outdir = os.path.join(options.db, 'out/saccade_view_joint_analysis')
db = FlydraDB(options.db, False)
set_namespace('saccade_view_joint_analysis')
# for each image we do a different report
data = {}
for image in images:
# For each image we have different tables
tables = ["saccades_view_%s_%s" % (view.id, image.id) for view in views]
all_available = [x for x in db.list_samples() if db.has_saccades(x) and
all([db.has_table(x, table) for table in tables])]
# We further divide these in post and nopost
groups_samples = {
'posts':
filter(lambda s: db.get_attr(s, 'stimulus') != 'nopost', all_available),
'noposts':
filter(lambda s: db.get_attr(s, 'stimulus') == 'nopost', all_available)
}
# now, for each group
for group in groups:
is_hallucination = image.id.startswith('h')
white_arena = image.id.endswith('_w')
if (not is_hallucination) and white_arena and (group.id == 'noposts'):
# if there are not posts, it's useless
continue
samples = groups_samples[group.id]
if not samples:
print "Warning: no samples for %s/%s" % (image.id, group.id)
continue
# global statistics
key = (group.id, image.id)
job_id = "%s-%s" % key
data[key] = comp(compute_stats, options.db,
samples, image.id, job_id=job_id)
for saccades_set, direction in prod(saccades_sets, dirs):
view2result = {}
for i, view in enumerate(views):
table = tables[i]
key = Exp(image=image.id, group=group.id,
view=view.id, dir=direction.id,
saccades_set=saccades_set.id)
job_id = "%s-%s-%s-%s-%s" % key
result = comp(compute_saccade_stats, options.db,
samples, table,
[direction.args, saccades_set.args],
job_id=job_id)
data[key] = result
view2result[view.id] = result
page_id = make_page_id(image=image.id, group=group.id,
dir=direction.id, saccades_set=saccades_set.id)
comp(render_page, view2result, outdir, page_id, job_id=page_id)
for saccades_set in saccades_sets:
table = "saccades_view_start_%s" % (image.id)
exp_id = '%s_%s_%s' % (image.id, group.id, saccades_set.id)
results = comp(bet_on_flies, options.db, samples, table, saccades_set,
job_id='lasvegas-' + exp_id + '-bet')
page_id = exp_id
comp(las_vegas_report, os.path.join(outdir, 'lasvegas'), page_id, results,
job_id='lasvegas-' + exp_id + '-report')
db.close()
comp(add_comparisons, data, outdir)
filename = os.path.join(outdir, 'gui.html')
comp(create_gui_new, filename, menus)
if options.interactive:
# start interactive session
compmake_console()
else:
# batch mode
# try to do everything
batch_command('make all')
# exit with error if we are not done
# (that is, make all failed for some reason)
todo = list(parse_job_list('todo'))
if todo:
logger.info('Still %d jobs to do.' % len(todo))
sys.exit(-2)
