def init(self): if self.config.db is not None and self.config.sample is not None: db = FlydraDB(self.config.db) stimulus_xml = db.get_attr(self.config.sample, 'stimulus_xml') self.arena_info = get_posts_info(stimulus_xml) else: self.arena_info = None
class FlydraImage(Generator): ''' This block outputs the retinal images from a FlydraDB for a particular sample. ''' Block.alias('flydra_db_image') Block.config('db', 'FlydraDB database directory') Block.config('sample', 'Sample ID -- such as "DATA20080611_191809".') Block.config('image', 'Which retinal image to display.') Block.output('obj_id') Block.output('frame') Block.output('image') def init(self): self.db = FlydraDB(self.config.db) if not self.db.has_sample(self.config.sample): raise ValueError('Sample "%s" not found.' % self.config.sample) if not self.db.has_table(self.config.sample, self.config.image): raise ValueError('Table "%s" not found for sample %s.' % \ (self.config.image, self.config.sample)) self.data = self.db.get_table(self.config.sample, self.config.image) self.next_index = 0 if len(self.data) == 0: self.info('Empty rows for sample %s.' % self.config.sample) self.next_index = None def update(self): row = self.data[self.next_index] t = row['time'] for field in ['obj_id', 'frame']: self.set_output(field, value=row[field], timestamp=t) self.set_output('image', row['value'], timestamp=t) self.next_index += 1 if self.next_index == len(self.data): self.next_index = None def next_data_status(self): # TODO: put new interface if self.next_index is None: # EOF return (False, None) else: return (True, self.data[self.next_index]['time'])
def main(): parser = OptionParser() parser.add_option("--db", help="FlydraDB directory") (options, args) = parser.parse_args() #@UnusedVariable if not options.db: raise Exception('Please define the FlydraDB directory using `--db`.') db = FlydraDB(options.db) choose = { 'andrea_detector': { 'Dpseudoobscura': 'threshold16', 'Dananassae': 'threshold10', 'Dhydei': 'threshold7', 'Dmelanogaster': 'threshold1', 'Darizonae': 'threshold9', 'Dmojavensis': 'threshold7', 'peter': 'peters_conf', 'mamaramaposts': 'use_for_report', 'mamaramanoposts': 'use_for_report' }, 'ros_detector': { 'Dpseudoobscura': 'filt_kalman-amp_th_10-th_4', 'Dananassae': 'filt_kalman-amp_th_10-th_4', 'Dhydei': 'filt_kalman-amp_th_10-th_4', 'Dmelanogaster': 'filt_kalman-amp_th_10-th_4', 'Darizonae': 'filt_kalman-amp_th_10-th_4', 'Dmojavensis': 'filt_kalman-amp_th_10-th_4' } } for official, choices in choose.items(): for group, version in choices.items(): print("Group %r: %r -> %r" % (group, version, official)) samples = db.list_samples_for_group(group) for sample in samples: print(" sample %r" % sample) table = db.get_table(sample, SACCADES_TABLE, version=version) copy = numpy.array(table, dtype=table.dtype) db.set_table(sample=sample, table=SACCADES_TABLE, version=official, data=copy) db.release_table(table) db.close()
def main(): parser = OptionParser(usage=description) parser.add_option("--db", default='flydra_db', help="FlydraDB directory") parser.add_option("--interactive", help="Start compmake interactive session." " Otherwise run in batch mode", default=False, action="store_true") (options, args) = parser.parse_args() #@UnusedVariable db = FlydraDB(options.db, False) set_namespace('video_contrast') samples = db.list_samples() if not samples: print 'No samples found' for id in samples: if db.has_rows(id) and db.has_table(id, 'contrast') and \ db.has_table(id, 'luminance'): config = {'sample': id, 'db': options.db} comp(pg, 'flydra_display_contrast', config, job_id="flydra_display_contrast:%s" % id) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: print('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def main(): set_namespace('env_stats') parser = OptionParser() parser.add_option("--db", default='flydra_db', help="FlydraDB directory") (options, args) = parser.parse_args() #@UnusedVariable db = FlydraDB(options.db, False) outdir = os.path.join(options.db, 'out/environment_stats') images = ["luminance", "contrast", "luminance_w", "contrast_w", "hluminance_w", "hcontrast_w"] for image in images: samples = [x for x in db.list_samples() if db.get_attr(x, 'stimulus', None) != 'nopost' and db.has_table(x, image)] if not samples: print "No samples for %s" % samples continue comp_prefix(image) data = comp(compute_environment_autocorrelation, options.db, samples, image) comp(create_report, data, image, outdir) db.close() compmake_console()
def main(): parser = OptionParser(usage=description) parser.add_option("--db", default='flydra_db', help="FlydraDB directory") (options, args) = parser.parse_args() #@UnusedVariable if options.db is None: logger.error('Please specify a directory using --db.') sys.exit(-1) db = FlydraDB(options.db) for sample in db.list_samples(): if not db.has_rows(sample): continue rows = db.get_rows(sample) v = rows[:]['linear_velocity_modulus'] perc = [1, 5, 95, 99] print "Sample %s" % sample print " linear_velocity_modulus" for p in perc: s = scipy.stats.scoreatpercentile(v, p) print ' - %2d%% %.3f m/s' % (p, s) db.release_table(rows)
class FlydraData(Generator): ''' This block outputs the data from a FlydraDB for a particular sample. ''' Block.alias('flydra_db_source') Block.config('db', 'Database directory') Block.config('sample', 'Sample ID -- such as "DATA20080611_191809".') for f in fields: Block.output(f) Block.output('stimulus_xml') def init(self): self.db = FlydraDB(self.config.db, False) self.rows = self.db.get_rows(self.config.sample) self.next_index = 0 def update(self): row = self.rows[self.next_index] t = row['time'] for field in fields: self.set_output(field, value=row[field].copy(), timestamp=t) if self.db.has_attr(self.config.sample, 'stimulus_xml'): stim_xml = self.db.get_attr(self.config.sample, 'stimulus_xml') self.set_output('stimulus_xml', stim_xml, timestamp=t) self.next_index += 1 if self.next_index == len(self.rows): self.next_index = None def next_data_status(self): # TODO: put new interface if self.next_index is None: # EOF return (False, None) else: return (True, self.rows[self.next_index]['time'])
def init(self): self.db = FlydraDB(self.config.db) if not self.db.has_sample(self.config.sample): raise ValueError('Sample "%s" not found.' % self.config.sample) if not self.db.has_table(self.config.sample, self.config.image): raise ValueError('Table "%s" not found for sample %s.' % \ (self.config.image, self.config.sample)) self.data = self.db.get_table(self.config.sample, self.config.image) self.next_index = 0 if len(self.data) == 0: self.info('Empty rows for sample %s.' % self.config.sample) self.next_index = None
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--image", default="luminance", help="Rendered image to use -- " " corresponding to image 'saccades_view_{start,stop}_X'") parser.add_option("--interactive", help="Start an interactive compmake session." " Otherwise run in batch mode. ", default=False, action="store_true") (options, args) = parser.parse_args() #@UnusedVariable if options.db is None: logger.error('Please specify a directory using --db.') sys.exit(-1) view_start = 'saccades_view_start_%s' % options.image view_stop = 'saccades_view_stop_%s' % options.image view_rstop = 'saccades_view_rstop_%s' % options.image db = FlydraDB(options.db, False) # all samples with enough data all_available = lambda x: db.has_saccades(x) and \ db.has_table(x, view_start) and \ db.has_table(x, view_stop) and \ db.has_table(x, view_rstop) samples = filter(all_available, db.list_samples()) set_namespace('saccade_view_show_%s' % options.image) for sample in samples: comp_prefix(sample) comp(create_and_write_report, options.db, sample, options.image) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def get_all_data_for_signal(db, samples, interval_function, signal, signal_component): db = FlydraDB(db, False) all = [] for id in samples: if not db.has_rows(id): logger.warning('Could not find rows table for %s; skipping.' % (id)) continue rows_table = db.get_rows(id) try: interval = interval_function(db, id, rows_table) except Exception as e: logger.warning('Cannot compute interval for sample %s: %s '\ % (id, e)) db.release_table(rows_table) continue rows = rows_table[interval] s = extract_signal(rows, signal, signal_component) all.append(s) db.release_table(rows_table) db.close() return numpy.concatenate(all)
def compute_mean_generic(db, samples, image, operator): ''' db: FlydraDB directory samples: list of IDs ''' db = FlydraDB(db, False) results = { 'samples': {} } ex = Expectation() for i, id in enumerate(samples): progress('Computing mean %s' % image, (i, len(samples)), "Sample %s" % id) if not (db.has_sample(id) and db.has_table(id, image)): raise ValueError('No table "%s" for id %s' % (image, id)) data = db.get_table(id, image) values = data[:]['value'] this = operator(values) # print "id: %s len: %d %d" % (id, len(data), len(values)) ex.update(this, len(data)) results['samples'][id] = this db.release_table(data) results['all'] = ex.get_value() db.close() return results
def compute_environment_autocorrelation(db, samples, image, maxlag=50): nsensors = 1398 results = numpy.ndarray(shape=(nsensors, 2 * maxlag + 1)) db = FlydraDB(db, create=False) block_size = 50 num_blocks = int(numpy.ceil(nsensors * 1.0 / block_size)) for b in range(num_blocks): start = block_size * b stop = min(start + block_size, nsensors) progress('Computing autocorrelation', (b, num_blocks)) data = [[] for i in range(nsensors)] for k, sample in enumerate(samples): progress('getting data', (k, len(samples)), sample) table = db.get_table(sample, image) chunk = (table[:]['value'][:, start:stop]).copy() for j, i in enumerate(range(start, stop)): data[i].append(chunk[:, j]) db.release_table(table) for j, i in enumerate(range(start, stop)): progress('Computing correlation', (j, stop - start)) x = numpy.concatenate(data[i]) corr, lags = xcorr(x, maxlag=maxlag) assert(len(lags) == 2 * maxlag + 1) results[i, :] = corr db.close() data = { 'results': results, 'lags': lags } return data
def main(): parser = OptionParser(usage=description) parser.add_option("--out", help="Output data directory", default="flydra2ros") parser.add_option("--db", help="Location of input Flydra db.") (options, args) = parser.parse_args() # @UnusedVariable if not options.db: raise Exception("Please define FlydraDB directory using `--db`.") verbose = True def printv(s): if verbose: print(s) db = FlydraDB(options.db, create=False) configuration = "use_for_report" for sample in db.list_samples(): if not db.has_table(sample, table=SACCADES_TABLE, version=configuration): continue group = guess_group(db, sample) magno = {} table = db.get_table(sample, SACCADES_TABLE, configuration) species = db.get_attr(sample, "species", "Dmelanogaster") magno["species"] = species magno["sample"] = sample # _name if db.has_table(sample, EXP_DATA_TABLE): exp_data = db.get_table(sample, EXP_DATA_TABLE) print(exp_data.dtype) timestamp = exp_data[:]["timestamp"] else: timestamp = None magno["use_for_report"] = convert_saccades_to_ros(table, timestamp) db.release_table(table) if timestamp is not None: db.release_table(exp_data) output_dir = os.path.join(options.out, group) filename = os.path.join(output_dir, "magno_%s.mat" % sample) printv("writing to %s" % filename) if not os.path.exists(output_dir): os.makedirs(output_dir) scipy.io.savemat(filename, {"magno": magno}, oned_as="row") # put species and sample print("closing") db.close()
def main(): parser = OptionParser(usage=description) parser.add_option("--saccade_data", help="Main data directory", default='saccade_data') parser.add_option("--db", help="FlydraDB directory") parser.add_option("--verbose", help='Verbose output', default=False, action="store_true") (options, args) = parser.parse_args() #@UnusedVariable if not options.db: raise Exception('Please define FlydraDB directory using `--db`.') def printv(s): if options.verbose: print(s) flydra_db = FlydraDB(options.db, create=True) matlab_dir = options.saccade_data for group in os.listdir(matlab_dir): group_dir = os.path.join(matlab_dir, group) if not os.path.isdir(group_dir): continue printv("Opening {0}".format(group)) # # # exp_data, attributes = read_raw_data(filename) # # consider_importing_processed(flydra_db, sample, exp_data, attributes) # # flydra_db.set_attr(sample, 'species', attributes['species']) # flydra_db.set_attr(sample, 'background', attributes['background']) # # flydra_db.set_table(sample, EXP_DATA_TABLE, exp_data) # flydra_db.add_sample_to_group(sample, group) # flydra_db.add_sample_to_group(sample, 'ros') # processed_dir = os.path.join(group_dir, 'processed') if not os.path.exists(processed_dir): printv("No processed data found for group %r." % group) continue for conf in os.listdir(processed_dir): # first look for saccades.mat saccades_file = os.path.join(processed_dir, conf, 'saccades.mat') if os.path.exists(saccades_file): printv('Loading from file %r.' % saccades_file) saccades = saccades_read_mat(saccades_file) samples = numpy.unique(saccades['sample']) for sample in samples: if not flydra_db.has_sample(sample): flydra_db.add_sample(sample) flydra_db.add_sample_to_group(sample, group) sample_saccades = saccades[saccades[:]['sample'] == sample] flydra_db.set_table(sample=sample, table=SACCADES_TABLE, version=conf, data=sample_saccades) # else: # prefix = 'data_' # suffix = '.mat' # for file in [file for file in os.listdir(group_dir) # if (file.startswith(prefix)) and file.endswith(suffix)]: # # sample = file[len(prefix):file.index('.')] # # if verbose: # print(" - Considering sample {0}".format(sample.__repr__())) # # if not flydra_db.has_sample(sample): # flydra_db.add_sample(sample) # # filename = os.path.join(group_dir, file) # # # # else: # for conf in os.listdir(processed_dir): # saccades = os.path.join(processed_dir, conf, 'saccades.mat') # if os.path.exists(saccades): # group_record.configurations[conf] = saccades # # add to general list # self.configurations.add(conf) ## else: ## conf_dir = os.path.join(processed_dir, conf) ## for file in [file for file in os.listdir(conf_dir) ## if file.startswith('processed_data_') and file.endswith('.mat')]: ## id = file[5:-7] # # # if we don't have exp data, get list of samples from # # processed data # if group_record.configurations and \ # not group_record.has_experimental_data: # saccades = saccades_read_mat(saccades) # group_record.samples = set(numpy.unique(saccades['sample'])) # for sample in group_record.samples: # self.sample2group[sample] = group # # if len(group_record.samples)> 0: # self.groups[group] = group_record # # print "has it", group, group_record.has_experimental_data # flydra_db.close()
def init(self): self.db = FlydraDB(self.config.db, False) self.rows = self.db.get_rows(self.config.sample) self.next_index = 0
def main(): parser = OptionParser(usage=description) parser.add_option("--db", default='flydra_db', help="FlydraDB directory") parser.add_option("--nocache", help="Ignores already computed results.", default=False, action="store_true") parser.add_option("--white", help="Computes luminance_w, with the arena" " painted white.", default=False, action="store_true") parser.add_option("--host", help="Use a remote rfsee. Otherwise," "use local process.", default=None) (options, args) = parser.parse_args() if options.db is None: logger.error('Please specify a directory using --db.') sys.exit(-1) db = FlydraDB(options.db) if args: do_samples = args else: # look for samples with the rows table all_samples = db.list_samples() do_samples = filter(lambda x: db.has_saccades(x) and db.has_attr(x, 'stimulus_xml'), all_samples) logger.info('Found %d/%d samples with saccades and stimulus info.' % (len(do_samples), len(all_samples))) image = 'luminance_w' if options.white else 'luminance' target_start = 'saccades_view_start_%s' % image target_stop = 'saccades_view_stop_%s' % image target_rstop = 'saccades_view_rstop_%s' % image target_sstop = 'saccades_view_sstop_%s' % image target_random = 'saccades_view_random_%s' % image for i, sample_id in enumerate(do_samples): logger.info('Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id)) if not db.has_sample(sample_id): raise Exception('Sample %s not found in db.' % sample_id) if not db.has_saccades(sample_id): raise Exception('Sample %s does not have saccades table.' % sample_id) if not db.has_attr(sample_id, 'stimulus_xml'): raise Exception('Sample %s does not have the stimulus' ' information ("stimulus_xml")' % sample_id) # todo: check stale dependencies if db.has_table(sample_id, target_start) and \ db.has_table(sample_id, target_stop) and \ db.has_table(sample_id, target_rstop) and \ db.has_table(sample_id, target_sstop) and \ db.has_table(sample_id, target_random) and \ not options.nocache: logger.info('Targets already computed for %s; skipping' % sample_id) continue # Get the stimulus description stimulus_xml = db.get_attr(sample_id, 'stimulus_xml') saccades = db.get_saccades(sample_id) view_start, view_stop, view_rstop, view_random, view_sstop = \ render_saccades_view( saccades=saccades, stimulus_xml=stimulus_xml, host=options.host, white=options.white) db.set_table(sample_id, target_start, view_start) db.set_table(sample_id, target_stop, view_stop) db.set_table(sample_id, target_rstop, view_rstop) db.set_table(sample_id, target_random, view_random) db.set_table(sample_id, target_sstop, view_sstop) db.release_table(saccades) db.close()
def main(): parser = OptionParser(usage=description) parser.add_option("--saccade_data", help="Main data directory", default='saccade_data') parser.add_option("--db", help='Location of output Flydra db.') (options, args) = parser.parse_args() #@UnusedVariable if not options.db: raise Exception('Please define FlydraDB directory using `--db`.') verbose = True flydra_db = FlydraDB(options.db, create=True) matlab_dir = options.saccade_data for group in os.listdir(matlab_dir): group_dir = os.path.join(matlab_dir, group) if not os.path.isdir(group_dir): continue if verbose: print("Opening {0}".format(group)) for file in [file for file in os.listdir(group_dir) if (file.startswith('magno_')) \ and file.endswith('.mat')]: sample = file[file.index('_') + 1:file.index('.')] if verbose: print(" - Considering sample {0}".format(sample.__repr__())) if not flydra_db.has_sample(sample): flydra_db.add_sample(sample) flydra_db.add_sample_to_group(sample, group) # flydra_db.add_sample_to_group(sample, 'ros') filename = os.path.join(group_dir, file) exp_data, attributes = read_raw_data(filename) consider_importing_processed(flydra_db, sample, exp_data, attributes) flydra_db.set_attr(sample, 'species', attributes['species']) flydra_db.set_attr(sample, 'background', attributes['background']) flydra_db.set_table(sample, EXP_DATA_TABLE, exp_data) flydra_db.close()
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="FlydraDB directory") parser.add_option("--nocache", help="Ignores already computed results.", default=False, action="store_true") parser.add_option("--compute_mu", help="Computes mu and optic flow.", default=False, action="store_true") parser.add_option("--white", help="Computes luminance_w, with the arena" " painted white.", default=False, action="store_true") parser.add_option("--host", help="Use a remote rfsee. Otherwise, use local process.", default=None) (options, args) = parser.parse_args() #@UnusedVariable db = FlydraDB(options.db, False) # look for samples with the rows table do_samples = db.list_samples() do_samples = filter(lambda x: db.has_rows(x) and db.get_attr(x, 'stimulus') == 'nopost', do_samples) if not do_samples: raise Exception('Cannot find samples to hallucinate about.') print "Summary, including nopost." for s in sorted(get_db_stimulus_stats(db, include_nopost=True), key=(lambda x:-x.total_length)): print "stimulus: {s.stimulus:>10} samples: {s.total_number:>5} "\ " total length: {len:>5} minutes".format(s=s, len=s.total_length / (60 * 60)) stimulus_to_use = list(get_stimulus_to_use(db, len(do_samples))) for i, sample in enumerate(do_samples): stimulus = stimulus_to_use[i][0] print sample, stimulus if options.white: target = 'hluminance_w' else: target = 'hluminance' for i, sample_id in enumerate(do_samples): stimulus = stimulus_to_use[i][0] stimulus_xml = stimulus_to_use[i][1] print 'Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id) if not db.has_sample(sample_id): raise Exception('Sample %s not found in db.' % sample_id) if not db.has_rows(sample_id): raise Exception('Sample %s does not have rows table.' % sample_id) if options.compute_mu: if db.has_table(sample_id, 'nearness') and not options.nocache: logger.info('Already computed nearness for %s; skipping' % sample_id) continue else: if db.has_table(sample_id, target) and not options.nocache: logger.info('Already computed luminance for %s; skipping' % sample_id) continue rows = db.get_rows(sample_id) results = render(rows, stimulus_xml, host=options.host, compute_mu=options.compute_mu, white=options.white) db.set_table(sample_id, target, results['luminance']) if options.compute_mu: db.set_table(sample_id, 'hnearness', results['nearness']) db.set_table(sample_id, 'hretinal_velocities', results['retinal_velocities']) db.release_table(rows) db.close()
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="FlydraDB directory") parser.add_option("--nocache", help="Ignores already computed results.", default=False, action="store_true") parser.add_option("--compute_mu", help="Computes mu and optic flow.", default=False, action="store_true") parser.add_option("--white", help="Computes luminance_w, with the arena" " painted white.", default=False, action="store_true") parser.add_option("--host", help="Use a remote rfsee. Otherwise, use local process.", default=None) (options, args) = parser.parse_args() db = FlydraDB(options.db, False) if args: do_samples = args else: # look for samples with the rows table do_samples = db.list_samples() do_samples = filter(lambda x: db.has_rows(x), do_samples) if options.white: target = 'luminance_w' else: target = 'luminance' for i, sample_id in enumerate(do_samples): print 'Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id) if not db.has_sample(sample_id): raise Exception('Sample %r not found in db.' % sample_id) if not db.has_rows(sample_id): raise Exception('Sample %r does not have rows table.' % sample_id) if not db.has_attr(sample_id, 'stimulus_xml'): raise Exception('Sample %r does not have the "stimulus_xml" attribute.' %sample_id) if options.compute_mu: if db.has_table(sample_id, 'nearness') and not options.nocache: logger.info('Already computed nearness for %r; skipping' % sample_id) continue else: if db.has_table(sample_id, target) and not options.nocache: logger.info('Already computed luminance for %r; skipping' % sample_id) continue rows = db.get_rows(sample_id) stimulus_xml = db.get_attr(sample_id, 'stimulus_xml') results = render(rows, stimulus_xml, host=options.host, compute_mu=options.compute_mu, white=options.white) db.set_table(sample_id, target, results['luminance']) if options.compute_mu: db.set_table(sample_id, 'nearness', results['nearness']) db.set_table(sample_id, 'retinal_velocities', results['retinal_velocities']) db.release_table(rows) db.close()
def open_flydra_db(self): self.flydra_db = FlydraDB(os.path.join(self.data, 'sac_flydra_db'))
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="FlydraDB directory") parser.add_option("--nocache", help="Ignores already computed results.", default=False, action="store_true") parser.add_option("--sigma", help="Kernel spread (degrees)", type="float", default=6) parser.add_option("--source", default='luminance', help="Source table") parser.add_option("--target", default='contrast', help="Destination table") (options, args) = parser.parse_args() kernel = get_contrast_kernel(sigma_deg=options.sigma, eyes_interact=False) kernel = kernel.astype('float32').copy('C') db = FlydraDB(options.db, False) if args: do_samples = args else: do_samples = db.list_samples() do_samples = filter(lambda x: db.has_table(x, options.source), do_samples) if not do_samples: raise Exception('No samples with table "%s" found. ' % options.source) for i, sample_id in enumerate(do_samples): logger.info('Sample %s/%s: %s' % (i + 1, len(do_samples), sample_id)) if not db.has_sample(sample_id): raise Exception('Sample %s not found in db.' % sample_id) if not db.has_table(sample_id, options.source): raise Exception('Sample %s does not have table %s; skipping.' \ % (sample_id, options.source)) if db.has_table(sample_id, options.target) and not options.nocache: logger.info('Already computed "%s" for %s; skipping' % \ (options.target, sample_id)) continue luminance = db.get_table(sample_id, options.source) contrast = compute_contrast_for_table(luminance, kernel) db.set_table(sample_id, options.target, contrast) db.release_table(luminance) db.close()
class SamplesDB: def __init__(self, data, verbose=False): ''' data: base directory ''' if not os.path.exists(data) or not os.path.isdir(data): raise Exception('Could not open directory %s' % data) self.data = data # self.use_cache = True self.use_cache = False self.use_flydra_db = True if self.use_cache: self.open_shelve() if self.use_flydra_db: self.open_flydra_db() self.groups = {} #self.group2samples = {} # maps id to .mat file self.sample2expmat = {} # maps id to .pickle file self.sample2exppickle = {} # list of all configurations self.configurations = set() # maps sample -> group self.sample2group = {} #print "Loading data in %s" % data for group in os.listdir(data): group_dir = os.path.join(data, group) if not os.path.isdir(group_dir): continue # print "Reading group %s" % group group_record = Group() for file in [file for file in os.listdir(group_dir) if file.startswith('data_') and file.endswith('.mat')]: id = file[5:-4] group_record.samples.add(id) self.sample2expmat[id] = os.path.join(group_dir,file) self.sample2group[id] = group for file in [file for file in os.listdir(group_dir) if file.startswith('data_') and file.endswith('.pickle')]: id = file[5:-7] group_record.samples.add(id) self.sample2exppickle[id] = os.path.join(group_dir,file) self.sample2group[id] = group group_record.has_experimental_data = len(group_record.samples) > 0 processed_dir = os.path.join(group_dir, 'processed') if not os.path.exists(processed_dir): if verbose: print "No processed data found for %s." % group pass else: for conf in os.listdir(processed_dir): saccades = os.path.join(processed_dir, conf, 'saccades.mat') if os.path.exists(saccades): group_record.configurations[conf] = saccades # add to general list self.configurations.add(conf) # else: # conf_dir = os.path.join(processed_dir, conf) # for file in [file for file in os.listdir(conf_dir) # if file.startswith('processed_data_') and file.endswith('.mat')]: # id = file[5:-7] # if we don't have exp data, get list of samples from # processed data if group_record.configurations and \ not group_record.has_experimental_data: saccades = saccades_read_mat(saccades) group_record.samples = set(numpy.unique(saccades['sample'])) for sample in group_record.samples: self.sample2group[sample] = group if len(group_record.samples)> 0: self.groups[group] = group_record print "has it", group, group_record.has_experimental_data def open_shelve(self): shelve_fname = os.path.join(self.data, 'shelve') self.shelve = shelve.open(shelve_fname, protocol=pickle.HIGHEST_PROTOCOL) def open_flydra_db(self): self.flydra_db = FlydraDB(os.path.join(self.data, 'sac_flydra_db')) def list_groups(self): """ Returns a list of the groups. """ return natsorted(list(self.groups.keys())) def list_all_samples(self): """ Returns a list of all samples for all groups. """ return natsorted(list(self.sample2group.keys())) def list_samples(self, group): """ Lists the samples in the given group. """ return natsorted(list(self.groups[group].samples)) def list_all_configurations(self): """ Lists all the configurations present in the data. """ return natsorted(self.configurations) def list_configurations(self, group): """ Lists the configurations for the given group. """ return natsorted(list(self.groups[group].configurations.keys())) def get_group_for_sample(self, sample): """ Returns the sample associated to the group. """ return self.sample2group[sample] def get_saccades_for_group(self, group, configuration): """ Returns the saccades for the given group and configuration. If configuration is not passed, we use the default. """ if self.use_flydra_db: table = 'groupsaccades_%s' % configuration if self.flydra_db.has_sample(group) and \ self.flydra_db.has_table(group, table): t = self.flydra_db.get_table(group, table) #value = t.copy() value = t #self.flydra_db.release_table(t) return value if self.use_cache: key = str(('get_saccades_for_group', group, configuration)) if key in self.shelve: return self.shelve[key] filename = self.groups[group].configurations[configuration] saccades = saccades_read_mat(filename) if self.use_flydra_db: if not self.flydra_db.has_sample(group): self.flydra_db.add_sample(group) self.flydra_db.set_table(group, table, saccades) if self.use_cache: self.shelve[key] = saccades return saccades def group_has_experimental_data(self, group): """ Returns true if this group has the raw orientation data. (mamarama has only saccades data. ) """ return self.groups[group].has_experimental_data def has_experimental_data(self, sample): """ Returns true if this sample has the raw orientation data. """ return sample in self.sample2expmat or sample in self. sample2exppickle def get_saccades_for_sample(self, sample, configuration): """ Returns the saccades for the given group and configuration. If configuration is not passed, we use the default. """ if self.use_flydra_db: table = 'saccades_%s' % configuration if self.flydra_db.has_sample(sample) and \ self.flydra_db.has_table(sample, table): t = self.flydra_db.get_table(sample, table) #value = t.copy() value = t #self.flydra_db.release_table(t) return value if self.use_cache: key = str(('get_saccades_for_sample', sample, configuration)) if key in self.shelve: return self.shelve[key] group = self.get_group_for_sample(sample) group_saccades = self.get_saccades_for_group(group, configuration) print group_saccades[0].dtype # with open('tmp.pickle','w') as f: # pickle.dump(f, group_saccades) mine = group_saccades[:]['sample'] == sample saccades = group_saccades[mine] if len(saccades) == 0: raise Exception('No saccades found for %s' % sample) if self.use_flydra_db: if not self.flydra_db.has_sample(sample): self.flydra_db.add_sample(sample) self.flydra_db.set_table(sample, table, saccades) if self.use_cache: self.shelve[key] = saccades return saccades def get_experimental_data(self, sample): # if self.use_flydra_db: # table = 'tethered_data' # if self.flydra_db.has_sample(sample) and \ # self.flydra_db.has_table(sample, table): # t = self.flydra_db.get_table(sample, table) # #value = t.copy() # value = t # #self.flydra_db.release_table(t) # return value # if self.use_cache: if sample in self.shelve: return self.shelve[sample] if sample in self.sample2expmat: data = scipy.io.loadmat(self.sample2expmat[sample], squeeze_me=True) data = data['data'] # convert from array to hash assert isinstance(data, numpy.ndarray) data = dict(map(lambda field: (field, data[field]), data.dtype.fields)) # convert from array to string for k in list(data.keys()): if data[k].dtype.char == 'U': data[k] = str(data[k]) # make sure everything is 1d array def as1d(x): if x.dtype == 'object': x = x.tolist() return x.reshape(len(x)) data['exp_orientation'] = as1d(data['exp_orientation']) data['exp_timestamps'] = as1d(data['exp_timestamps']) elif sample in self.sample2exppickle: with open(self.sample2exppickle[sample], 'rb') as f: data = cPickle.load(f) else: raise Exception('no data for sample %s found' % sample) # if self.use_flydra_db: # if not self.flydra_db.has_sample(sample): # self.flydra_db.add_sample(sample) # # self.flydra_db.set_table(sample, table, data) if self.use_cache: self.shelve[sample] = data return data def __getstate__(self): # do not pickle the shelve all = dict(self.__dict__) all['shelve'] = None all['flydra_db'] = None return all
def main(): parser = OptionParser(usage=description) parser.add_option("--flydra_db", default="saccade_data_flydradb", help="Main data directory") parser.add_option( "--interactive", action="store_true", default=False, help="Starts an interactive compmake session." ) parser.add_option("--report", default="saccade_report", help="Saccade report directory") parser.add_option("--groups", default=None, help="Which groups to consider") parser.add_option("--configurations", default=None, help="Which configurations to consider") parser.add_option("--combid", default=None, help="How to name this combination of groups/configs.") (options, args) = parser.parse_args() # @UnusedVariable if args: raise Exception("Spurious arguments %r." % args) db = FlydraDB(options.flydra_db) robust_split = lambda s: filter(lambda x: x, s.split(",")) if not options.groups in [None, "all"]: groups = robust_split(options.groups) if not groups: raise Exception("No groups specified.") groupset = "_".join(groups) else: groups = db.list_groups() groupset = "all" if not groups: raise Exception("No groups found.") if not options.configurations in [None, "all"]: configurations = robust_split(options.configurations) if not configurations: raise Exception("No configuration specified") confset = "_".join(configurations) else: configurations = db.list_all_versions_for_table(SACCADES_TABLE) confset = "all" configurations = set() for group in groups: configurations.update(db.list_versions_for_table_in_group(group, SACCADES_TABLE)) configurations = natsorted(configurations) if not configurations: raise Exception("No valid versions of table %r found." % SACCADES_TABLE) print ("I will consider the configurations: %r" % configurations) if options.combid is None: combination = "%s_%s" % (groupset, confset) else: combination = options.combid print ("I call this combination %r." % combination) output_dir = os.path.join(options.report, combination) if not os.path.exists(output_dir): os.makedirs(output_dir) set_namespace("master_plot_%s" % combination) # we maintain several indices # key = (group, configuration, plot) index_group_plots = {} # key = (sample, plot) index_sample_expdata_plots = {} # key = (sample, configuration, plot) index_sample_saccades_plots = {} # First we index the DB print ("Looking for data in database...") all_samples = set() group2samples = {} configurations_for_group = {} group_has_exp_data = {} for group in groups: group2samples[group] = db.list_samples_for_group(group) all_samples.update(group2samples[group]) if not group2samples[group]: raise Exception("Empty group %r." % group) available = db.list_versions_for_table_in_group(group=group, table=SACCADES_TABLE) configurations_for_group[group] = set(configurations).intersection(available) if not configurations_for_group[group]: print ("No configurations found for %r; available %r" % (group, available)) group_has_exp_data[group] = db.group_has_table(group, EXP_DATA_TABLE) all_samples = natsorted(all_samples) # print info print ("Summary:") for group in groups: msg = " group {group:>20} samples: {nsamples:3} " " configurations: {nconf:3} raw data? {data}".format( group=group, nsamples=len(group2samples[group]), nconf=len(configurations_for_group[group]), data=group_has_exp_data[group], ) print (msg) # TODO: iterate by sample, not by group for group in groups: for configuration in configurations_for_group[group]: for plot in group_plots: job_id = "%s-%s-%s" % (group, configuration, plot.id) index_group_plots[(group, configuration, plot.id)] = comp( wrap_group_plot, options.flydra_db, group, configuration, plot.command, plot.args, job_id=job_id ) for sample, plot in prod(group2samples[group], sample_saccades_plots): job_id = "%s-%s-%s" % (sample, configuration, plot.id) key = (sample, configuration, plot.id) if key in index_sample_saccades_plots: # we already did it as part of another group continue index_sample_saccades_plots[key] = comp( wrap_sample_saccades_plot, options.flydra_db, sample, configuration, plot.command, plot.args, job_id=job_id, ) if group_has_exp_data[group]: for sample, plot in prod(group2samples[group], sample_expdata_plots): job_id = "%s-%s" % (sample, plot.id) key = (sample, plot.id) if key in index_sample_expdata_plots: # we already did it as part of another group continue index_sample_expdata_plots[key] = comp( wrap_sample_expdata_plot, options.flydra_db, sample, plot.command, plot.args, job_id=job_id ) # now we create the indices # fix configuration, function; iterate groups for configuration, plot in itertools.product(configurations, group_plots): subs = [] descs = [] page_id = "%s.%s" % (configuration, plot.id) for group, group_desc in order_groups(groups): if not configuration in configurations_for_group[group]: continue descs.append(group_desc) subs.append(index_group_plots[(group, configuration, plot.id)]) if not subs: raise Exception("no groups for configuration %r." % configuration) job_id = page_id comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id) comp( create_gui, filename=os.path.join(output_dir, "group_plots.html"), menus=[ ("Detector", configurations, configurations), ("Plot/table", map(lambda x: x.id, group_plots), map(lambda x: x.description, group_plots)), ], job_id="gui-group_plots", ) # fix group, function; iterate samples for group in groups: if not group_has_exp_data[group]: continue for plot in sample_expdata_plots: subs = [] descs = [] for sample in group2samples[group]: descs.append(sample) subs.append(index_sample_expdata_plots[(sample, plot.id)]) page_id = "%s.%s" % (group, plot.id) job_id = page_id comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id) # get the ordered group lists and desc ordered_groups = map(lambda t: t[0], order_groups(groups)) ordered_groups_desc = map(lambda t: t[1], order_groups(groups)) comp( create_gui, filename=os.path.join(output_dir, "expdata_plots.html"), menus=[ ("Group", ordered_groups, ordered_groups_desc), ( "Plot/table", map(lambda x: x.id, sample_expdata_plots), map(lambda x: x.description, sample_expdata_plots), ), ], job_id="gui-expdata_plots", ) # fix configuration, group, function; iterate samples for group in groups: for configuration in configurations: if not configuration in configurations_for_group[group]: for plot in sample_saccades_plots: page_id = "%s.%s.%s" % (configuration, group, plot.id) comp( write_empty, page_id, output_dir, "Group %r has not been processed with algorithm %r." % (group, configuration), job_id=page_id, ) continue for plot in sample_saccades_plots: subs = [] descs = [] for sample in group2samples[group]: descs.append(sample) r = index_sample_saccades_plots[(sample, configuration, plot.id)] subs.append(r) page_id = "%s.%s.%s" % (configuration, group, plot.id) job_id = page_id comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id) comp( create_gui, filename=os.path.join(output_dir, "saccade_plots.html"), menus=[ ("Detector", configurations, configurations), ("Group", ordered_groups, ordered_groups_desc), ( "Plot/table", map(lambda x: x.id, sample_saccades_plots), map(lambda x: x.description, sample_saccades_plots), ), ], job_id="gui-saccade_plots", ) # fix configuration, sample; plot fullsscreen for group in groups: for configuration in configurations: for sample in group2samples[group]: # XXX make it clenaer if not configuration in configurations_for_group[group]: for plot in sample_fullscreen_plots: page_id = "%s.%s.%s" % (sample, configuration, plot.id) comp( write_empty, page_id, output_dir, 'Group %s has not been processed with algorithm "%s".' % (group, configuration), job_id=page_id, ) # print "skipping sample %s group %s config %s" %\ # (sample,group, configuration) continue if not group_has_exp_data[group]: for plot in sample_fullscreen_plots: page_id = "%s.%s.%s" % (sample, configuration, plot.id) comp( write_empty, page_id, output_dir, "Group %r does not have raw experimental data." % (group), job_id=page_id, ) continue for plot in sample_fullscreen_plots: job_id = "%s-%s-%s" % (sample, configuration, plot.id) # FIXME: error if sample in 2 groups job = comp( wrap_sample_saccades_plot, options.flydra_db, sample, configuration, plot.command, plot.args, job_id=job_id, ) page_id = "%s.%s.%s" % (sample, configuration, plot.id) comp(write_report, job, output_dir, page_id, job_id=job_id + "-write_report") comp( create_gui, filename=os.path.join(output_dir, "sample_fullscreen_plots.html"), menus=[ ("Sample", all_samples, all_samples), ("Detector", configurations, configurations), ( "Plot/table", map(lambda x: x.id, sample_fullscreen_plots), map(lambda x: x.description, sample_fullscreen_plots), ), ], job_id="gui-sample_fullscreen_plots", ) tabs = [ ( "group_plots", "By group", "This set displays one plot/table for each group of samples. " "You have the further choice of detection algorithm and plot/table to display.", ), ( "saccade_plots", "By sample", "This set displays one plot/table for each individual sample. " "You have the further choice of which group to consider, which " "detection algorithm, and which plot/table to display.", ), ( "expdata_plots", "By sample (raw)", "This set displays one plot/table for each individual sample, " " produced from the raw data (no saccade detection, so no choice of detector). " "You have the further choice of which group to consider, " "and which plot/table to display." " Note that some samples might be missing; for example, we don't use " " raw orientation data for the Mamarama samples.", ), ( "sample_fullscreen_plots", "By sample, single page", "This set displays one entire page for each sample. " "You have the further choice of sample, " "detection algorithm, and which plot/table to display.", ), ] comp(create_main_gui, tabs, filename=os.path.join(output_dir, "main.html"), job_id="gui-main") if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command("make all") # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list("todo")) if todo: print ("Still %d jobs to do." % len(todo)) sys.exit(-2)
def main_filter(args): parser = LenientOptionParser() parser.add_option("--db", default='flydra_db', help="FlydraDB directory") parser.add_option("--min_frames_per_track", default=400, help="Minimum number of frames per track [= %default]") parser.add_option("--confirm_problems", help="Stop interactively on problems with log files'\ '(e.g.: cannot find valid obj_ids) [default: %default]", default=False, action="store_true") parser.add_option("--dynamic_model_name", help="Smoothing dynamical model [default: %default]", default="mamarama, units: mm") parser.add_option("--debug_output", help="Creates debug figures.", default=False, action="store_true") parser.add_option("--nocache", help="Ignores already computed results.", default=False, action="store_true") parser.add_option("--smoothing", help="Uses Kalman-smoothed data.", default=False, action="store_true") (options, args) = parser.parse_args(args) table_name = 'rows' # TODO: use constant table_version = "smooth" if options.smoothing else "kf" if not args: raise UserError('No files or directories specified.') if not os.path.exists(options.db): os.makedirs(options.db) db = FlydraDB(options.db) good_files = get_good_files(where=args, pattern="*.kh5", confirm_problems=options.confirm_problems) if len(good_files) == 0: logger.error("No good files to process") sys.exit(1) n = len(good_files) for i in range(n): (filename, obj_ids, stim_fname) = good_files[i] logger.info('Sample %s/%s: %s' % (i + 1, n, filename)) # only maintain basename stim = os.path.splitext(os.path.basename(stim_fname))[0] sample_id = os.path.splitext(os.path.basename(filename))[0] logger.info("File %d/%d %s %s %s " % (i, n, str(filename), str(obj_ids), stim_fname)) if (db.has_sample(sample_id) and db.has_table(sample_id, table_name, table_version) and not options.nocache): logger.info('Sample %r already computed; skipping.' ' (use --nocache to ignore)' % sample_id) continue all_data = [] for obj_id, rows in get_good_smoothed_tracks(#@UnusedVariable filename=filename, obj_ids=obj_ids, min_frames_per_track=options.min_frames_per_track, dynamic_model_name=options.dynamic_model_name, use_smoothing=options.smoothing): filtered = filter_rows(rows, options) all_data.append(filtered) if not all_data: logger.info('Not enough data found for %r; skipping.' % sample_id) continue if not db.has_sample(sample_id): db.add_sample(sample_id) db.set_attr(sample_id, 'stim_fname', stim_fname) db.set_attr(sample_id, 'stimulus', stim) stim_xml = open(stim_fname).read() db.set_attr(sample_id, 'stimulus_xml', stim_xml) geometry = get_posts_info(stim_xml) print(geometry) db.set_attr(sample_id, 'posts', geometry['posts']) if 'arena' in geometry: db.set_attr(sample_id, 'arena', geometry['arena']) db.add_sample_to_group(sample_id, stim) if stim != 'nopost': db.add_sample_to_group(sample_id, 'posts') rows = numpy.concatenate(all_data) db.set_table(sample=sample_id, table=table_name, data=rows, version=table_version) db.set_attr(sample_id, 'filter_time', datetime.now().strftime("%Y%m%d_%H%M%S")) db.set_attr(sample_id, 'filter_host', platform.node()) db.set_attr(sample_id, 'filter_user', get_user()) db.set_attr(sample_id, 'filter_python_version', platform.python_version()) db.set_attr(sample_id, 'filter_numpy_version', numpy.version.version) db.close()
def main(): parser = OptionParser(usage=description) parser.add_option( "--db", default='flydra_db_directory', help="FlydraDB directory") parser.add_option("--model", help="ProcGraph model name.") parser.add_option( "--needs", help="Comma-separated list of tables required", default="rows,luminance") parser.add_option( "--interactive", help="Start compmake interactive session." " Otherwise run in batch mode", default=False, action="store_true") (options, args) = parser.parse_args() if options.model is None: print "Please specify the model." sys.exit(-3) print("Using FlydraDB directory %r." % options.db) db = FlydraDB(options.db, False) # TODO: make the storage inside options.db? set_namespace('run_pg_model_%s' % options.model) tables = options.needs.split(',') if args: samples = args for sample in samples: if not db.has_sample(sample): raise Exception('Unknown sample %r' % sample) else: samples = db.list_samples() if not samples: print 'No samples found' num_ok = 0 for id in samples: enough = all(map(lambda t: db.has_table(id, t), tables)) if not enough: continue num_ok += 1 config = {'sample': id, 'db': options.db} comp(pg, options.model, config, job_id=id) logger.info( "Found %d/%d samples with tables %s." % (num_ok, len(samples), tables)) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--image", help="Which image to plot.") parser.add_option("--filter", help="Which procgraph filter to use to plot.", default="flydra_simple_video_filter") parser.add_option("--interactive", help="Start compmake interactive session." " Otherwise run in batch mode", default=False, action="store_true") (options, args) = parser.parse_args() if options.image is None: print "Usage: %s [--db DB] --image <image> [ids]" % sys.argv[0] sys.exit(-1) db = FlydraDB(options.db) set_namespace('video_image_%s' % options.image) if args: samples = args else: # look for samples with the rows table samples = db.list_samples() samples = filter(lambda x: db.has_table(x, options.image), samples) if not samples: raise Exception('No samples found at all with available image "%s".' % \ options.image) for id in samples: if not db.has_table(id, options.image): raise Exception('Sample %s does not have table "%s".' % (id, options.image)) config = {'sample': id, 'db': options.db, 'image': options.image, 'filter': options.filter} comp(pg, 'flydra_simple_video', config, job_id="%s" % id) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: print('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def create_and_write_report(flydra_db, sample, image_name): view_start = 'saccades_view_start_%s' % image_name view_stop = 'saccades_view_stop_%s' % image_name view_rstop = 'saccades_view_rstop_%s' % image_name db = FlydraDB(flydra_db, False) saccades = db.get_saccades(sample) values_start = db.get_table(sample, view_start) values_stop = db.get_table(sample, view_stop) values_rstop = db.get_table(sample, view_rstop) r = Report(sample) for i in range(len(saccades)): ri = r.node("saccade-%04d" % i) ri.data_rgb('start', plot_luminance(values_start[i]['value'])) ri.data_rgb('stop', plot_luminance(values_stop[i]['value'])) ri.data_rgb('rstop', plot_luminance(values_rstop[i]['value'])) f = ri.figure(shape=(1, 3)) f.sub('start', 'At saccade start') f.sub('stop', 'At saccade stop') #f.sub('rstop', '(random stop)') db.release_table(saccades) db.release_table(values_start) db.release_table(values_stop) db.release_table(values_rstop) filename = "%s/out/saccade_view_show/%s_%s.html" % (flydra_db, image_name, sample) print "Writing to %s" % filename r.to_html(filename)
def main(): parser = OptionParser(usage=description) parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--interactive", default=False, action="store_true", help="Start a compmake interactive session." " Otherwise run in batch mode") parser.add_option("--empty_group_ok", default=False, action="store_true", help="do not give up if one group does not have samples ") (options, args) = parser.parse_args() #@UnusedVariable if options.db is None: logger.error('Please specify a directory using --db.') sys.exit(-1) outdir = os.path.join(options.db, 'out/saccade_view_joint_analysis') db = FlydraDB(options.db, False) set_namespace('saccade_view_joint_analysis') # for each image we do a different report data = {} for image in images: # For each image we have different tables tables = ["saccades_view_%s_%s" % (view.id, image.id) for view in views] all_available = [x for x in db.list_samples() if db.has_saccades(x) and all([db.has_table(x, table) for table in tables])] # We further divide these in post and nopost groups_samples = { 'posts': filter(lambda s: db.get_attr(s, 'stimulus') != 'nopost', all_available), 'noposts': filter(lambda s: db.get_attr(s, 'stimulus') == 'nopost', all_available) } # now, for each group for group in groups: is_hallucination = image.id.startswith('h') white_arena = image.id.endswith('_w') if (not is_hallucination) and white_arena and (group.id == 'noposts'): # if there are not posts, it's useless continue samples = groups_samples[group.id] if not samples: print "Warning: no samples for %s/%s" % (image.id, group.id) continue # global statistics key = (group.id, image.id) job_id = "%s-%s" % key data[key] = comp(compute_stats, options.db, samples, image.id, job_id=job_id) for saccades_set, direction in prod(saccades_sets, dirs): view2result = {} for i, view in enumerate(views): table = tables[i] key = Exp(image=image.id, group=group.id, view=view.id, dir=direction.id, saccades_set=saccades_set.id) job_id = "%s-%s-%s-%s-%s" % key result = comp(compute_saccade_stats, options.db, samples, table, [direction.args, saccades_set.args], job_id=job_id) data[key] = result view2result[view.id] = result page_id = make_page_id(image=image.id, group=group.id, dir=direction.id, saccades_set=saccades_set.id) comp(render_page, view2result, outdir, page_id, job_id=page_id) for saccades_set in saccades_sets: table = "saccades_view_start_%s" % (image.id) exp_id = '%s_%s_%s' % (image.id, group.id, saccades_set.id) results = comp(bet_on_flies, options.db, samples, table, saccades_set, job_id='lasvegas-' + exp_id + '-bet') page_id = exp_id comp(las_vegas_report, os.path.join(outdir, 'lasvegas'), page_id, results, job_id='lasvegas-' + exp_id + '-report') db.close() comp(add_comparisons, data, outdir) filename = os.path.join(outdir, 'gui.html') comp(create_gui_new, filename, menus) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # exit with error if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)