def main():
parser = OptionParser(usage=description)
parser.add_option("--flydra_db", default="saccade_data_flydradb", help="Main data directory")
parser.add_option(
"--interactive", action="store_true", default=False, help="Starts an interactive compmake session."
)
parser.add_option("--report", default="saccade_report", help="Saccade report directory")
parser.add_option("--groups", default=None, help="Which groups to consider")
parser.add_option("--configurations", default=None, help="Which configurations to consider")
parser.add_option("--combid", default=None, help="How to name this combination of groups/configs.")
(options, args) = parser.parse_args() # @UnusedVariable
if args:
raise Exception("Spurious arguments %r." % args)
db = FlydraDB(options.flydra_db)
robust_split = lambda s: filter(lambda x: x, s.split(","))
if not options.groups in [None, "all"]:
groups = robust_split(options.groups)
if not groups:
raise Exception("No groups specified.")
groupset = "_".join(groups)
else:
groups = db.list_groups()
groupset = "all"
if not groups:
raise Exception("No groups found.")
if not options.configurations in [None, "all"]:
configurations = robust_split(options.configurations)
if not configurations:
raise Exception("No configuration specified")
confset = "_".join(configurations)
else:
configurations = db.list_all_versions_for_table(SACCADES_TABLE)
confset = "all"
configurations = set()
for group in groups:
configurations.update(db.list_versions_for_table_in_group(group, SACCADES_TABLE))
configurations = natsorted(configurations)
if not configurations:
raise Exception("No valid versions of table %r found." % SACCADES_TABLE)
print ("I will consider the configurations: %r" % configurations)
if options.combid is None:
combination = "%s_%s" % (groupset, confset)
else:
combination = options.combid
print ("I call this combination %r." % combination)
output_dir = os.path.join(options.report, combination)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
set_namespace("master_plot_%s" % combination)
# we maintain several indices
# key = (group, configuration, plot)
index_group_plots = {}
# key = (sample, plot)
index_sample_expdata_plots = {}
# key = (sample, configuration, plot)
index_sample_saccades_plots = {}
# First we index the DB
print ("Looking for data in database...")
all_samples = set()
group2samples = {}
configurations_for_group = {}
group_has_exp_data = {}
for group in groups:
group2samples[group] = db.list_samples_for_group(group)
all_samples.update(group2samples[group])
if not group2samples[group]:
raise Exception("Empty group %r." % group)
available = db.list_versions_for_table_in_group(group=group, table=SACCADES_TABLE)
configurations_for_group[group] = set(configurations).intersection(available)
if not configurations_for_group[group]:
print ("No configurations found for %r; available %r" % (group, available))
group_has_exp_data[group] = db.group_has_table(group, EXP_DATA_TABLE)
all_samples = natsorted(all_samples)
# print info
print ("Summary:")
for group in groups:
msg = " group {group:>20} samples: {nsamples:3} " " configurations: {nconf:3} raw data? {data}".format(
group=group,
nsamples=len(group2samples[group]),
nconf=len(configurations_for_group[group]),
data=group_has_exp_data[group],
)
print (msg)
# TODO: iterate by sample, not by group
for group in groups:
for configuration in configurations_for_group[group]:
for plot in group_plots:
job_id = "%s-%s-%s" % (group, configuration, plot.id)
index_group_plots[(group, configuration, plot.id)] = comp(
wrap_group_plot, options.flydra_db, group, configuration, plot.command, plot.args, job_id=job_id
)
for sample, plot in prod(group2samples[group], sample_saccades_plots):
job_id = "%s-%s-%s" % (sample, configuration, plot.id)
key = (sample, configuration, plot.id)
if key in index_sample_saccades_plots:
# we already did it as part of another group
continue
index_sample_saccades_plots[key] = comp(
wrap_sample_saccades_plot,
options.flydra_db,
sample,
configuration,
plot.command,
plot.args,
job_id=job_id,
)
if group_has_exp_data[group]:
for sample, plot in prod(group2samples[group], sample_expdata_plots):
job_id = "%s-%s" % (sample, plot.id)
key = (sample, plot.id)
if key in index_sample_expdata_plots:
# we already did it as part of another group
continue
index_sample_expdata_plots[key] = comp(
wrap_sample_expdata_plot, options.flydra_db, sample, plot.command, plot.args, job_id=job_id
)
# now we create the indices
# fix configuration, function; iterate groups
for configuration, plot in itertools.product(configurations, group_plots):
subs = []
descs = []
page_id = "%s.%s" % (configuration, plot.id)
for group, group_desc in order_groups(groups):
if not configuration in configurations_for_group[group]:
continue
descs.append(group_desc)
subs.append(index_group_plots[(group, configuration, plot.id)])
if not subs:
raise Exception("no groups for configuration %r." % configuration)
job_id = page_id
comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id)
comp(
create_gui,
filename=os.path.join(output_dir, "group_plots.html"),
menus=[
("Detector", configurations, configurations),
("Plot/table", map(lambda x: x.id, group_plots), map(lambda x: x.description, group_plots)),
],
job_id="gui-group_plots",
)
# fix group, function; iterate samples
for group in groups:
if not group_has_exp_data[group]:
continue
for plot in sample_expdata_plots:
subs = []
descs = []
for sample in group2samples[group]:
descs.append(sample)
subs.append(index_sample_expdata_plots[(sample, plot.id)])
page_id = "%s.%s" % (group, plot.id)
job_id = page_id
comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id)
# get the ordered group lists and desc
ordered_groups = map(lambda t: t[0], order_groups(groups))
ordered_groups_desc = map(lambda t: t[1], order_groups(groups))
comp(
create_gui,
filename=os.path.join(output_dir, "expdata_plots.html"),
menus=[
("Group", ordered_groups, ordered_groups_desc),
(
"Plot/table",
map(lambda x: x.id, sample_expdata_plots),
map(lambda x: x.description, sample_expdata_plots),
),
],
job_id="gui-expdata_plots",
)
# fix configuration, group, function; iterate samples
for group in groups:
for configuration in configurations:
if not configuration in configurations_for_group[group]:
for plot in sample_saccades_plots:
page_id = "%s.%s.%s" % (configuration, group, plot.id)
comp(
write_empty,
page_id,
output_dir,
"Group %r has not been processed with algorithm %r." % (group, configuration),
job_id=page_id,
)
continue
for plot in sample_saccades_plots:
subs = []
descs = []
for sample in group2samples[group]:
descs.append(sample)
r = index_sample_saccades_plots[(sample, configuration, plot.id)]
subs.append(r)
page_id = "%s.%s.%s" % (configuration, group, plot.id)
job_id = page_id
comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id)
comp(
create_gui,
filename=os.path.join(output_dir, "saccade_plots.html"),
menus=[
("Detector", configurations, configurations),
("Group", ordered_groups, ordered_groups_desc),
(
"Plot/table",
map(lambda x: x.id, sample_saccades_plots),
map(lambda x: x.description, sample_saccades_plots),
),
],
job_id="gui-saccade_plots",
)
# fix configuration, sample; plot fullsscreen
for group in groups:
for configuration in configurations:
for sample in group2samples[group]:
# XXX make it clenaer
if not configuration in configurations_for_group[group]:
for plot in sample_fullscreen_plots:
page_id = "%s.%s.%s" % (sample, configuration, plot.id)
comp(
write_empty,
page_id,
output_dir,
'Group %s has not been processed with algorithm "%s".' % (group, configuration),
job_id=page_id,
)
# print "skipping sample %s group %s config %s" %\
# (sample,group, configuration)
continue
if not group_has_exp_data[group]:
for plot in sample_fullscreen_plots:
page_id = "%s.%s.%s" % (sample, configuration, plot.id)
comp(
write_empty,
page_id,
output_dir,
"Group %r does not have raw experimental data." % (group),
job_id=page_id,
)
continue
for plot in sample_fullscreen_plots:
job_id = "%s-%s-%s" % (sample, configuration, plot.id)
# FIXME: error if sample in 2 groups
job = comp(
wrap_sample_saccades_plot,
options.flydra_db,
sample,
configuration,
plot.command,
plot.args,
job_id=job_id,
)
page_id = "%s.%s.%s" % (sample, configuration, plot.id)
comp(write_report, job, output_dir, page_id, job_id=job_id + "-write_report")
comp(
create_gui,
filename=os.path.join(output_dir, "sample_fullscreen_plots.html"),
menus=[
("Sample", all_samples, all_samples),
("Detector", configurations, configurations),
(
"Plot/table",
map(lambda x: x.id, sample_fullscreen_plots),
map(lambda x: x.description, sample_fullscreen_plots),
),
],
job_id="gui-sample_fullscreen_plots",
)
tabs = [
(
"group_plots",
"By group",
"This set displays one plot/table for each group of samples. "
"You have the further choice of detection algorithm and plot/table to display.",
),
(
"saccade_plots",
"By sample",
"This set displays one plot/table for each individual sample. "
"You have the further choice of which group to consider, which "
"detection algorithm, and which plot/table to display.",
),
(
"expdata_plots",
"By sample (raw)",
"This set displays one plot/table for each individual sample, "
" produced from the raw data (no saccade detection, so no choice of detector). "
"You have the further choice of which group to consider, "
"and which plot/table to display."
" Note that some samples might be missing; for example, we don't use "
" raw orientation data for the Mamarama samples.",
),
(
"sample_fullscreen_plots",
"By sample, single page",
"This set displays one entire page for each sample. "
"You have the further choice of sample, "
"detection algorithm, and which plot/table to display.",
),
]
comp(create_main_gui, tabs, filename=os.path.join(output_dir, "main.html"), job_id="gui-main")
if options.interactive:
# start interactive session
compmake_console()
else:
# batch mode
# try to do everything
batch_command("make all")
# start the console if we are not done
# (that is, make all failed for some reason)
todo = list(parse_job_list("todo"))
if todo:
print ("Still %d jobs to do." % len(todo))
sys.exit(-2)
