def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--image", default="luminance", help="Rendered image to use -- " " corresponding to image 'saccades_view_{start,stop}_X'") parser.add_option("--interactive", help="Start an interactive compmake session." " Otherwise run in batch mode. ", default=False, action="store_true") (options, args) = parser.parse_args() #@UnusedVariable if options.db is None: logger.error('Please specify a directory using --db.') sys.exit(-1) view_start = 'saccades_view_start_%s' % options.image view_stop = 'saccades_view_stop_%s' % options.image view_rstop = 'saccades_view_rstop_%s' % options.image db = FlydraDB(options.db, False) # all samples with enough data all_available = lambda x: db.has_saccades(x) and \ db.has_table(x, view_start) and \ db.has_table(x, view_stop) and \ db.has_table(x, view_rstop) samples = filter(all_available, db.list_samples()) set_namespace('saccade_view_show_%s' % options.image) for sample in samples: comp_prefix(sample) comp(create_and_write_report, options.db, sample, options.image) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def main(): parser = OptionParser(usage=description) parser.add_option("--db", default='flydra_db', help="FlydraDB directory") parser.add_option("--interactive", help="Start compmake interactive session." " Otherwise run in batch mode", default=False, action="store_true") (options, args) = parser.parse_args() #@UnusedVariable db = FlydraDB(options.db, False) set_namespace('video_contrast') samples = db.list_samples() if not samples: print 'No samples found' for id in samples: if db.has_rows(id) and db.has_table(id, 'contrast') and \ db.has_table(id, 'luminance'): config = {'sample': id, 'db': options.db} comp(pg, 'flydra_display_contrast', config, job_id="flydra_display_contrast:%s" % id) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: print('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def main(): parser = OptionParser(usage=description) parser.add_option("--flydra_db", default="saccade_data_flydradb", help="Main data directory") parser.add_option( "--interactive", action="store_true", default=False, help="Starts an interactive compmake session." ) parser.add_option("--report", default="saccade_report", help="Saccade report directory") parser.add_option("--groups", default=None, help="Which groups to consider") parser.add_option("--configurations", default=None, help="Which configurations to consider") parser.add_option("--combid", default=None, help="How to name this combination of groups/configs.") (options, args) = parser.parse_args() # @UnusedVariable if args: raise Exception("Spurious arguments %r." % args) db = FlydraDB(options.flydra_db) robust_split = lambda s: filter(lambda x: x, s.split(",")) if not options.groups in [None, "all"]: groups = robust_split(options.groups) if not groups: raise Exception("No groups specified.") groupset = "_".join(groups) else: groups = db.list_groups() groupset = "all" if not groups: raise Exception("No groups found.") if not options.configurations in [None, "all"]: configurations = robust_split(options.configurations) if not configurations: raise Exception("No configuration specified") confset = "_".join(configurations) else: configurations = db.list_all_versions_for_table(SACCADES_TABLE) confset = "all" configurations = set() for group in groups: configurations.update(db.list_versions_for_table_in_group(group, SACCADES_TABLE)) configurations = natsorted(configurations) if not configurations: raise Exception("No valid versions of table %r found." % SACCADES_TABLE) print ("I will consider the configurations: %r" % configurations) if options.combid is None: combination = "%s_%s" % (groupset, confset) else: combination = options.combid print ("I call this combination %r." % combination) output_dir = os.path.join(options.report, combination) if not os.path.exists(output_dir): os.makedirs(output_dir) set_namespace("master_plot_%s" % combination) # we maintain several indices # key = (group, configuration, plot) index_group_plots = {} # key = (sample, plot) index_sample_expdata_plots = {} # key = (sample, configuration, plot) index_sample_saccades_plots = {} # First we index the DB print ("Looking for data in database...") all_samples = set() group2samples = {} configurations_for_group = {} group_has_exp_data = {} for group in groups: group2samples[group] = db.list_samples_for_group(group) all_samples.update(group2samples[group]) if not group2samples[group]: raise Exception("Empty group %r." % group) available = db.list_versions_for_table_in_group(group=group, table=SACCADES_TABLE) configurations_for_group[group] = set(configurations).intersection(available) if not configurations_for_group[group]: print ("No configurations found for %r; available %r" % (group, available)) group_has_exp_data[group] = db.group_has_table(group, EXP_DATA_TABLE) all_samples = natsorted(all_samples) # print info print ("Summary:") for group in groups: msg = " group {group:>20} samples: {nsamples:3} " " configurations: {nconf:3} raw data? {data}".format( group=group, nsamples=len(group2samples[group]), nconf=len(configurations_for_group[group]), data=group_has_exp_data[group], ) print (msg) # TODO: iterate by sample, not by group for group in groups: for configuration in configurations_for_group[group]: for plot in group_plots: job_id = "%s-%s-%s" % (group, configuration, plot.id) index_group_plots[(group, configuration, plot.id)] = comp( wrap_group_plot, options.flydra_db, group, configuration, plot.command, plot.args, job_id=job_id ) for sample, plot in prod(group2samples[group], sample_saccades_plots): job_id = "%s-%s-%s" % (sample, configuration, plot.id) key = (sample, configuration, plot.id) if key in index_sample_saccades_plots: # we already did it as part of another group continue index_sample_saccades_plots[key] = comp( wrap_sample_saccades_plot, options.flydra_db, sample, configuration, plot.command, plot.args, job_id=job_id, ) if group_has_exp_data[group]: for sample, plot in prod(group2samples[group], sample_expdata_plots): job_id = "%s-%s" % (sample, plot.id) key = (sample, plot.id) if key in index_sample_expdata_plots: # we already did it as part of another group continue index_sample_expdata_plots[key] = comp( wrap_sample_expdata_plot, options.flydra_db, sample, plot.command, plot.args, job_id=job_id ) # now we create the indices # fix configuration, function; iterate groups for configuration, plot in itertools.product(configurations, group_plots): subs = [] descs = [] page_id = "%s.%s" % (configuration, plot.id) for group, group_desc in order_groups(groups): if not configuration in configurations_for_group[group]: continue descs.append(group_desc) subs.append(index_group_plots[(group, configuration, plot.id)]) if not subs: raise Exception("no groups for configuration %r." % configuration) job_id = page_id comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id) comp( create_gui, filename=os.path.join(output_dir, "group_plots.html"), menus=[ ("Detector", configurations, configurations), ("Plot/table", map(lambda x: x.id, group_plots), map(lambda x: x.description, group_plots)), ], job_id="gui-group_plots", ) # fix group, function; iterate samples for group in groups: if not group_has_exp_data[group]: continue for plot in sample_expdata_plots: subs = [] descs = [] for sample in group2samples[group]: descs.append(sample) subs.append(index_sample_expdata_plots[(sample, plot.id)]) page_id = "%s.%s" % (group, plot.id) job_id = page_id comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id) # get the ordered group lists and desc ordered_groups = map(lambda t: t[0], order_groups(groups)) ordered_groups_desc = map(lambda t: t[1], order_groups(groups)) comp( create_gui, filename=os.path.join(output_dir, "expdata_plots.html"), menus=[ ("Group", ordered_groups, ordered_groups_desc), ( "Plot/table", map(lambda x: x.id, sample_expdata_plots), map(lambda x: x.description, sample_expdata_plots), ), ], job_id="gui-expdata_plots", ) # fix configuration, group, function; iterate samples for group in groups: for configuration in configurations: if not configuration in configurations_for_group[group]: for plot in sample_saccades_plots: page_id = "%s.%s.%s" % (configuration, group, plot.id) comp( write_empty, page_id, output_dir, "Group %r has not been processed with algorithm %r." % (group, configuration), job_id=page_id, ) continue for plot in sample_saccades_plots: subs = [] descs = [] for sample in group2samples[group]: descs.append(sample) r = index_sample_saccades_plots[(sample, configuration, plot.id)] subs.append(r) page_id = "%s.%s.%s" % (configuration, group, plot.id) job_id = page_id comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id) comp( create_gui, filename=os.path.join(output_dir, "saccade_plots.html"), menus=[ ("Detector", configurations, configurations), ("Group", ordered_groups, ordered_groups_desc), ( "Plot/table", map(lambda x: x.id, sample_saccades_plots), map(lambda x: x.description, sample_saccades_plots), ), ], job_id="gui-saccade_plots", ) # fix configuration, sample; plot fullsscreen for group in groups: for configuration in configurations: for sample in group2samples[group]: # XXX make it clenaer if not configuration in configurations_for_group[group]: for plot in sample_fullscreen_plots: page_id = "%s.%s.%s" % (sample, configuration, plot.id) comp( write_empty, page_id, output_dir, 'Group %s has not been processed with algorithm "%s".' % (group, configuration), job_id=page_id, ) # print "skipping sample %s group %s config %s" %\ # (sample,group, configuration) continue if not group_has_exp_data[group]: for plot in sample_fullscreen_plots: page_id = "%s.%s.%s" % (sample, configuration, plot.id) comp( write_empty, page_id, output_dir, "Group %r does not have raw experimental data." % (group), job_id=page_id, ) continue for plot in sample_fullscreen_plots: job_id = "%s-%s-%s" % (sample, configuration, plot.id) # FIXME: error if sample in 2 groups job = comp( wrap_sample_saccades_plot, options.flydra_db, sample, configuration, plot.command, plot.args, job_id=job_id, ) page_id = "%s.%s.%s" % (sample, configuration, plot.id) comp(write_report, job, output_dir, page_id, job_id=job_id + "-write_report") comp( create_gui, filename=os.path.join(output_dir, "sample_fullscreen_plots.html"), menus=[ ("Sample", all_samples, all_samples), ("Detector", configurations, configurations), ( "Plot/table", map(lambda x: x.id, sample_fullscreen_plots), map(lambda x: x.description, sample_fullscreen_plots), ), ], job_id="gui-sample_fullscreen_plots", ) tabs = [ ( "group_plots", "By group", "This set displays one plot/table for each group of samples. " "You have the further choice of detection algorithm and plot/table to display.", ), ( "saccade_plots", "By sample", "This set displays one plot/table for each individual sample. " "You have the further choice of which group to consider, which " "detection algorithm, and which plot/table to display.", ), ( "expdata_plots", "By sample (raw)", "This set displays one plot/table for each individual sample, " " produced from the raw data (no saccade detection, so no choice of detector). " "You have the further choice of which group to consider, " "and which plot/table to display." " Note that some samples might be missing; for example, we don't use " " raw orientation data for the Mamarama samples.", ), ( "sample_fullscreen_plots", "By sample, single page", "This set displays one entire page for each sample. " "You have the further choice of sample, " "detection algorithm, and which plot/table to display.", ), ] comp(create_main_gui, tabs, filename=os.path.join(output_dir, "main.html"), job_id="gui-main") if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command("make all") # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list("todo")) if todo: print ("Still %d jobs to do." % len(todo)) sys.exit(-2)
def main(): parser = OptionParser(usage=description) parser.add_option( "--db", default='flydra_db_directory', help="FlydraDB directory") parser.add_option("--model", help="ProcGraph model name.") parser.add_option( "--needs", help="Comma-separated list of tables required", default="rows,luminance") parser.add_option( "--interactive", help="Start compmake interactive session." " Otherwise run in batch mode", default=False, action="store_true") (options, args) = parser.parse_args() if options.model is None: print "Please specify the model." sys.exit(-3) print("Using FlydraDB directory %r." % options.db) db = FlydraDB(options.db, False) # TODO: make the storage inside options.db? set_namespace('run_pg_model_%s' % options.model) tables = options.needs.split(',') if args: samples = args for sample in samples: if not db.has_sample(sample): raise Exception('Unknown sample %r' % sample) else: samples = db.list_samples() if not samples: print 'No samples found' num_ok = 0 for id in samples: enough = all(map(lambda t: db.has_table(id, t), tables)) if not enough: continue num_ok += 1 config = {'sample': id, 'db': options.db} comp(pg, options.model, config, job_id=id) logger.info( "Found %d/%d samples with tables %s." % (num_ok, len(samples), tables)) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--image", help="Which image to plot.") parser.add_option("--filter", help="Which procgraph filter to use to plot.", default="flydra_simple_video_filter") parser.add_option("--interactive", help="Start compmake interactive session." " Otherwise run in batch mode", default=False, action="store_true") (options, args) = parser.parse_args() if options.image is None: print "Usage: %s [--db DB] --image <image> [ids]" % sys.argv[0] sys.exit(-1) db = FlydraDB(options.db) set_namespace('video_image_%s' % options.image) if args: samples = args else: # look for samples with the rows table samples = db.list_samples() samples = filter(lambda x: db.has_table(x, options.image), samples) if not samples: raise Exception('No samples found at all with available image "%s".' % \ options.image) for id in samples: if not db.has_table(id, options.image): raise Exception('Sample %s does not have table "%s".' % (id, options.image)) config = {'sample': id, 'db': options.db, 'image': options.image, 'filter': options.filter} comp(pg, 'flydra_simple_video', config, job_id="%s" % id) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: print('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def main(): parser = OptionParser(usage=description) parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--interactive", default=False, action="store_true", help="Start a compmake interactive session." " Otherwise run in batch mode") parser.add_option("--empty_group_ok", default=False, action="store_true", help="do not give up if one group does not have samples ") (options, args) = parser.parse_args() #@UnusedVariable if options.db is None: logger.error('Please specify a directory using --db.') sys.exit(-1) outdir = os.path.join(options.db, 'out/saccade_view_joint_analysis') db = FlydraDB(options.db, False) set_namespace('saccade_view_joint_analysis') # for each image we do a different report data = {} for image in images: # For each image we have different tables tables = ["saccades_view_%s_%s" % (view.id, image.id) for view in views] all_available = [x for x in db.list_samples() if db.has_saccades(x) and all([db.has_table(x, table) for table in tables])] # We further divide these in post and nopost groups_samples = { 'posts': filter(lambda s: db.get_attr(s, 'stimulus') != 'nopost', all_available), 'noposts': filter(lambda s: db.get_attr(s, 'stimulus') == 'nopost', all_available) } # now, for each group for group in groups: is_hallucination = image.id.startswith('h') white_arena = image.id.endswith('_w') if (not is_hallucination) and white_arena and (group.id == 'noposts'): # if there are not posts, it's useless continue samples = groups_samples[group.id] if not samples: print "Warning: no samples for %s/%s" % (image.id, group.id) continue # global statistics key = (group.id, image.id) job_id = "%s-%s" % key data[key] = comp(compute_stats, options.db, samples, image.id, job_id=job_id) for saccades_set, direction in prod(saccades_sets, dirs): view2result = {} for i, view in enumerate(views): table = tables[i] key = Exp(image=image.id, group=group.id, view=view.id, dir=direction.id, saccades_set=saccades_set.id) job_id = "%s-%s-%s-%s-%s" % key result = comp(compute_saccade_stats, options.db, samples, table, [direction.args, saccades_set.args], job_id=job_id) data[key] = result view2result[view.id] = result page_id = make_page_id(image=image.id, group=group.id, dir=direction.id, saccades_set=saccades_set.id) comp(render_page, view2result, outdir, page_id, job_id=page_id) for saccades_set in saccades_sets: table = "saccades_view_start_%s" % (image.id) exp_id = '%s_%s_%s' % (image.id, group.id, saccades_set.id) results = comp(bet_on_flies, options.db, samples, table, saccades_set, job_id='lasvegas-' + exp_id + '-bet') page_id = exp_id comp(las_vegas_report, os.path.join(outdir, 'lasvegas'), page_id, results, job_id='lasvegas-' + exp_id + '-report') db.close() comp(add_comparisons, data, outdir) filename = os.path.join(outdir, 'gui.html') comp(create_gui_new, filename, menus) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # exit with error if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)