def test_order(): from compmake import comp, batch_command # make A -> B(fail) -> C A = comp(job_success, job_id='A') B = comp(job_failure, A, job_id='B') comp(job_success, B, job_id='C') batch_command('make') check_job_states(A=Cache.DONE, B=Cache.FAILED, C=Cache.BLOCKED)
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--image", default="luminance", help="Rendered image to use -- " " corresponding to image 'saccades_view_{start,stop}_X'") parser.add_option("--interactive", help="Start an interactive compmake session." " Otherwise run in batch mode. ", default=False, action="store_true") (options, args) = parser.parse_args() #@UnusedVariable if options.db is None: logger.error('Please specify a directory using --db.') sys.exit(-1) view_start = 'saccades_view_start_%s' % options.image view_stop = 'saccades_view_stop_%s' % options.image view_rstop = 'saccades_view_rstop_%s' % options.image db = FlydraDB(options.db, False) # all samples with enough data all_available = lambda x: db.has_saccades(x) and \ db.has_table(x, view_start) and \ db.has_table(x, view_stop) and \ db.has_table(x, view_rstop) samples = filter(all_available, db.list_samples()) set_namespace('saccade_view_show_%s' % options.image) for sample in samples: comp_prefix(sample) comp(create_and_write_report, options.db, sample, options.image) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def test_order(self): from compmake import comp, batch_command # add two copies comp(top, comp(bottom)) comp(top, comp(bottom)) batch_command('clean') batch_command('make') self.assertEqual(['bottom', 'top', 'bottom', 'top'], TestOrder.order)
def test_order_3(self): from compmake import comp, batch_command # choose wisely here comp(top, comp(bottom2)) comp(bottom) comp(top, comp(bottom2)) batch_command('clean') batch_command('make') self.assertEqual(['bottom', 'bottom2', 'top', 'bottom2', 'top'], TestOrder.order)
def test_execution_stats(self): from compmake import comp, batch_command # schedule some commands res = comp(f, comp(f), comp(f, comp(f))) result = compmake_execution_stats(res) batch_command('make') res = get_job_userobject(result.job_id) assert isinstance(res, dict) res['cpu_time'] res['wall_time'] res['jobs'] print res
def rlearn(config, parser): #@UnusedVariable t0 = time.time() """ Displays the learned DDS """ parser.add_option("-n", "--nthreads", help="Number of threads", type='int', default='4') parser.add_option("-r", "--nrefine", help="Number of time to refine learning", type='int', default='2') parser.add_option("-s", "--streams", help="Which streams to use.", default="*") parser.add_option("-i", "--comb", default="default") parser.add_option("-l", "--learners", help="Learner config.", default="*") parser.add_option("-o", "--output", help="Output directory", default='out/dp-rlearn/') parser.add_option("--sensels", default=None, help="Sensel indices to use for debuging refining module") parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") parser.add_option("--show", default=None, help="Name of learners to report") options = parser.parse_options() if options.show is not None: diffeomorphism2d_continuous.make_report(options.show.split(',')) sys.exit() nthreads = options.nthreads nrefine = options.nrefine learners = config.learners.expand_names(options.learners) streams = config.streams.expand_names(options.streams) if len(learners) > 1: logger.warn('Multiple learners are not supported for now') if len(streams) > 1: logger.warn('Multiple streams are not supported for now') id_comb = ",".join(streams) + "-" + ",".join(learners) outdir = os.path.join(options.output, id_comb) storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() rm = ReportManager(os.path.join(outdir, 'reports')) jobs_rlearn(config, rm, learners, streams, outdir, nthreads, nrefine, options.sensels) # Time and report the learning comp(learning_times_rlearn, outdir, learners, streams, nthreads, nrefine) rm.create_index_job() if options.command: return batch_command(options.command) else: compmake_console() return 0 logger.info("Done after time: " + str(time.time() - t0) + ' seconds')
def main(): # parser.add_option("--res", help="", default='[40, 80, 160, 320]') # options = parser.parse_options() rm = ReportManager('timeuse_reports') rm.add(comp(timeuse_report), 'timeuse', systems='all') rm.create_index_job() return batch_command('clean; make')
def dp_predstats_main(config, parser): parser.add_option("-o", "--output", default='out/dp-pred-stats', help="Output directory") parser.add_option("-d", "--distances", default='*', help="Comma-separated list of distances. Can use *.") parser.add_option("-S", "--dds", help="Comma-separated list of diffeosystems.") parser.add_option("-s", "--streams", help="Comma-separated list of streams.") parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") options = parser.parse_options() if not options.streams: msg = 'Please specify streams using -s.' raise UserError(msg) if not options.dds: msg = 'Please specify which discdds to use.' raise UserError(msg) distances = config.distances.expand_names(options.distances) streams = config.streams.expand_names(options.streams) dds = config.streams.expand_names(options.dds) logger.info('Using distances: %s' % distances) logger.info('Using streams: %s' % streams) logger.info('Using discdds: %s' % dds) outdir = '%s/%s' % (options.output, options.dds) storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() rm = ReportManager(os.path.join(outdir, "reports")) # for id_discdds in dds: create_predstats_jobs(config=config, distances=distances, id_discdds=dds, streams=streams, rm=rm, maxd=10) rm.create_index_job() if options.command: return batch_command(options.command) else: compmake_console() return 0
def main(): parser = OptionParser(usage=description) parser.add_option("--db", default='flydra_db', help="FlydraDB directory") parser.add_option("--interactive", help="Start compmake interactive session." " Otherwise run in batch mode", default=False, action="store_true") (options, args) = parser.parse_args() #@UnusedVariable db = FlydraDB(options.db, False) set_namespace('video_contrast') samples = db.list_samples() if not samples: print 'No samples found' for id in samples: if db.has_rows(id) and db.has_table(id, 'contrast') and \ db.has_table(id, 'luminance'): config = {'sample': id, 'db': options.db} comp(pg, 'flydra_display_contrast', config, job_id="flydra_display_contrast:%s" % id) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: print('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def pysnip_make(dirname, compmake_command): files = glob(os.path.join(dirname, '*.py')) prefixes = [os.path.splitext(os.path.basename(f))[0] for f in files] logger.info('Found %d snippets in directory %s' % (len(prefixes), dirname)) use_filesystem(os.path.join(dirname, '.compmake')) ntodo = 0 for p in prefixes: job = Job(dirname, p) job_id = job.basename current_state = None if job_exists(job_id): current_state = get_job_cache(job_id).state if job.status == DONE_UPTODATE: # logger.info('%s: done' % job.basename) if current_state != Cache.DONE: mark_as_done(job_id) pass elif job.status == FAILED: # logger.info('%s: failed' % job.basename) if current_state != Cache.FAILED: mark_as_failed(job_id) elif job.status == DONE_NEEDSUPDATE: mark_as_notstarted(job_id) # logger.info('%s: done (but needs update)' % job.basename) pass elif job.status == NOTSTARTED: mark_as_notstarted(job_id) # logger.info('%s: not started' % job.basename) pass comp(run_job, job, job_id=job_id) if job.status != DONE_UPTODATE: ntodo += 1 # logger.info('%d/%d jobs to do' % (ntodo, len(prefixes))) batch_command('stats') if compmake_command is not None: return batch_command(compmake_command) else: compmake_console() return 0
def dp_dist_stats(config, parser): #@UnusedVariable """ Computes statistics for images distances for different plan steps. """ parser.add_option("-o", "--output", default='out/dp-dist-stats', help="Output directory") parser.add_option("-d", "--distances", default='*', help="Comma-separated list of distances. Can use *.") parser.add_option("-s", "--streams", default='*', help="Comma-separated list of streams. Can use *.") parser.add_option("-r", "--repeat", default=1, type='int', help="Repeat many times.") parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") options = parser.parse_options() distances = natsorted(config.distances.expand_names(options.distances)) streams = natsorted(config.streams.expand_names(options.streams)) logger.info('Using distances: %s' % distances) logger.info('Using streams: %s' % streams) id_comb = ','.join(streams) + '-' + ','.join(distances) outdir = os.path.join(options.output, id_comb) storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() rm = ReportManager(os.path.join(outdir, "reports")) create_diststats_jobs(config=config, distances=distances, streams=streams, rm=rm, maxd=10) rm.create_index_job() if options.command: return batch_command(options.command) else: compmake_console() return 0
def dp_batch_main(config, parser): """ Runs batch planning experiments from batch configuration files. """ parser.add_option("-o", "--output", default='out/dp-batch', help="Output directory") parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") options, which = parser.parse() if not which: todo = config.sets.keys() id_comb = 'all' else: todo = config.sets.expand_names(which) id_comb = "+".join(sorted(todo)) logger.info('Batch sets to do: %s' % todo) outdir = os.path.join(options.output, 'set-%s' % id_comb) # Compmake storage for results storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() for id_set in todo: logger.info('Instantiating batch set %s' % id_set) spec = config.sets[id_set] try: algos = config.algos.expand_names(spec['algorithms']) testcases = config.testcases.expand_names(spec['testcases']) comp_prefix('%s' % id_set) b_outdir = os.path.join(outdir, id_set) create_bench_jobs(config=config, algos=algos, testcases=testcases, outdir=b_outdir) except: logger.error('Error while instantiating batch\n%s' % pformat(spec)) raise if options.command: return batch_command(options.command) else: compmake_console() return 0
def plearn(config, parser): """ Learn the diffeomorphisms in parallel. """ # parser.add_option("-i", "--id_image", help="ID image.", default='lena') ncpus = multiprocessing.cpu_count() parser.add_option("-n", "--nthreads", help="Number of threads", type='int', default=ncpus) parser.add_option("-s", "--streams", help="Which streams to use.", default="*") parser.add_option("-i", "--comb", default="default") parser.add_option("-l", "--learners", help="Learner config.", default="*") parser.add_option("-o", "--output", help="Output directory", default='out/dp-plearn/') parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") options = parser.parse_options() nthreads = options.nthreads learners = config.learners.expand_names(options.learners) streams = config.streams.expand_names(options.streams) id_comb = ",".join(streams) + "-" + ",".join(learners) outdir = os.path.join(options.output, id_comb) storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() rm = ReportManager(os.path.join(outdir, 'reports')) jobs_plearn(config, rm, learners, streams, outdir, nthreads) rm.create_index_job() # Time and report the learning comp(learning_times_plearn, outdir, learners, streams, nthreads) if options.command: return batch_command(options.command) else: compmake_console() return 0
def dp_bench_main(config, parser): # @UnusedVariable """ Runs a set of planning experiments. """ parser.add_option("-a", "--algorithms", default='*', help="Comma-separated list of algorithms. Can use *.") parser.add_option("-t", "--testcases", default='*', help="Comma-separated list of algorithms. Can use *.") parser.add_option("-o", "--output", default='out/dp-bench', help="Output directory") parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") options = parser.parse_options() algos = config.algos.expand_names(options.algorithms) testcases = config.testcases.expand_names(options.testcases) logger.info('Using %d algorithms: %s' % (len(algos), algos)) logger.info('Using %d testcases.' % (len(testcases))) outdir = options.output # Compmake storage for results storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() create_bench_jobs(config=config, algos=algos, testcases=testcases, outdir=outdir) if options.command: return batch_command(options.command) else: compmake_console() return 0
def uncert(config, parser): parser.add_option("-S", "--dds", help="DDS sytem .") parser.add_option("-c", "--command", \ help="Command to pass to compmake for batch mode") parser.add_option("-o", "--output", help="Output directory", \ default='out/dp-precision/') parser.add_option("-s", "--streams", help="Which streams to use.", \ default="*") parser.add_option("-d", "--distances", default='L2,L2w', \ help="Distances id to use for comparing the diffeo systems") parser.add_option("-l", "--length", default=9, type='int', \ help="Max length of test cases") options = parser.parse_options() dds = config.discdds.expand_names(options.dds) streams = config.streams.expand_names(options.streams) id_comb = ",".join(dds) outdir = os.path.join(options.output, id_comb) storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() rm = ReportManager(os.path.join(outdir, 'reports')) max_delta = options.length distances = config.distances.expand_names(options.distances) store = create_stats_jobs(config, dds, streams, distances, max_delta, outdir) records = comp(make_records, store) report = comp(report_stats, records, dds, streams, distances) r0 = comp(empty_report) rm.add(report, 'main', id_dds='dds') rm.add(r0, 'empty') rm.create_index_job() if options.command: return batch_command(options.command) else: compmake_console() return 0
def uncert(config, parser): # @UnusedVariable parser.add_option("-s", "--streams", help="Which streams to use.", default="*") parser.add_option("-S", "--dds", help="DDS sytem .") parser.add_option("-o", "--output", help="Output directory", default='out/dp-uncert/') parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") options = parser.parse_options() dds = config.discdds.expand_names(options.dds) streams = config.streams.expand_names(options.streams) id_comb = ",".join(dds) + "-" + ",".join(streams) outdir = os.path.join(options.output, id_comb) storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() rm = ReportManager(os.path.join(outdir, 'reports')) max_delta = 9 store = create_uncert_stats_jobs(config, dds, streams, max_delta, outdir) records = comp(make_records, store) report = comp(report_uncert_stats, records, dds) r0 = comp(empty_report) rm.add(report, 'main', id_dds='dds') rm.add(r0, 'empty') rm.create_index_job() if options.command: return batch_command(options.command) else: compmake_console() return 0
def idealize_uncert(config, parser): ''' Assumes a constant displacement over the whole sensor domain ''' parser.add_option("-S", "--dds", help="DDS sytem .") parser.add_option("-c", "--command", \ help="Command to pass to compmake for batch mode") parser.add_option("-o", "--output", help="Output directory", \ default='out/idealized-dds/') options = parser.parse_options() id_discdds = options.dds dds = config.discdds.instance(id_discdds) outdir = os.path.join(options.output, id_discdds) storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() # rm = ReportManager(os.path.join(outdir, 'reports')) # Ideal id_iu_discdds = 'iu-' + id_discdds iu_dds = comp(_idealize_uncert, dds, job_id='idealize_uncert') comp(save_results, id_iu_discdds, outdir, iu_dds) diffeo_report = comp(report_dds, 'iu_dds-%s' % id_discdds, iu_dds) rm.add(diffeo_report, 'iu-dds', id_learner='idealized-uncertainty') # Relative id_uur_discdds = 'uur-' + id_discdds dds_copyr = copy.copy(dds) uur_dds = comp(_update_uncert, dds_copyr, length_score_norm_relative, job_id='update_uncert_relative') comp(save_results, id_uur_discdds, outdir, uur_dds, job_id='update_uncert_relative_save') diffeo_report = comp(report_dds, 'uur-dds-%s' % id_discdds, uur_dds, job_id='update_uncert_relative_report') rm.add(diffeo_report, 'uur-dds', id_learner='updated-uncertainty-uur') # Absolute dds_copya = copy.copy(dds) id_uua_discdds = 'uua-' + id_discdds uua_dds = comp(_update_uncert, dds_copya, length_score_norm, job_id='update_uncert_absolute') comp(save_results, id_uua_discdds, outdir, uua_dds, job_id='update_uncert_absolute_save') diffeo_report = comp(report_dds, 'uua-dds-%s' % id_discdds, uua_dds, job_id='update_uncert_absolute_report') rm.add(diffeo_report, 'uua-dds', id_learner='updated-uncertainty-uua') rm.create_index_job() if options.command: return batch_command(options.command) else: # pdb.set_trace() compmake_console() return 0
def report_main(args): # np.seterr(all='raise') parser = LenientOptionParser(usage=description) parser.add_option("--db", help="Main data directory") parser.add_option("--outdir", help="Output directory for reports") parser.add_option("--datadir", help="Output directory for compmake files") parser.add_option("--version_rows", help="Table version ('kf' or 'smooth')") parser.add_option("--version_saccades", help="Table version ('kf', 'smooth', 'angvel')") parser.add_option("--group", help="Sample group", default='nopost') parser.add_option("--ncells_distance", type='int', default=20, help="Discretization for distance") parser.add_option("--ncells_axis_angle", type='int', default=36, help="Discretization for axis angle") parser.add_option("--compmake_command", default=None, help="Execute the CompMake command and exit.") parser.add_option("--pdf", default=False, action='store_true', help="Uses PDF for the reports (slower).") (options, args) = parser.parse_args(args) # @UnusedVariable check_no_spurious(args) check_mandatory(options, ['db', 'outdir', 'datadir', 'version_rows', 'version_saccades']) if options.pdf: logger.info('Using PDF for plots.') RepRepDefaults.default_image_format = MIME_PDF PlotParams.init_matplotlib() confid = '%s-%s-%s-D%d-A%d' % (options.group, options.version_rows, options.version_saccades, options.ncells_distance, options.ncells_axis_angle) compmake_dir = os.path.join(options.datadir, confid) use_filesystem(compmake_dir) logger.info('Storing computation in %r.' % compmake_dir) arena_radius = ParamsEstimation.arena_radius warnings.warn('Using hardcoded arena radius %s.' % arena_radius) cells = DACells( ncells_distance=options.ncells_distance, ncells_axis_angle=options.ncells_axis_angle, arena_radius=arena_radius, min_distance=ParamsEstimation.min_distance, bin_enlarge_angle=ParamsEstimation.bin_enlarge_angle, bin_enlarge_dist=ParamsEstimation.bin_enlarge_dist) stats = comp(get_group_density_stats, options.db, options.group, options.version_rows, cells) saccades = comp(get_saccades_for_group, options.db, options.group, options.version_saccades) saccades_stats = comp(compute_histogram_saccades, saccades, cells) joint_stats = comp(compute_joint_statistics, stats, saccades_stats) joint_stats = comp(compute_visual_stimulus, joint_stats) report = comp(report_stats, confid, stats, saccades_stats) rd = os.path.join(options.outdir, 'images') html = os.path.join(options.outdir, "%s.html" % confid) comp(write_report, report, html, rd, job_id='report_stats-write') report_m = comp(report_models_choice, confid, joint_stats) html = os.path.join(options.outdir, "%s_models.html" % confid) comp(write_report, report_m, html, rd, job_id='report_models_choice-write') report_s = comp(report_visual_stimulus, confid, joint_stats, job_id='report_stimulus') html = os.path.join(options.outdir, "%s_stimulus.html" % confid) comp(write_report, report_s, html, rd, job_id='report_stimulus-write') report_i = comp(report_intuitive, confid, joint_stats, job_id='report_intuitive') html = os.path.join(options.outdir, "%s_intuitive.html" % confid) comp(write_report, report_i, html, rd, job_id='report_intuitive-write') comp(write_report, comp(report_saccades, confid, saccades, job_id='report_saccades'), html=os.path.join(options.outdir, "%s_saccades.html" % confid), rd=rd, job_id='report_saccades-write') comp(write_report, comp(report_traj, confid, options.db, options.group, options.version_rows, job_id='report_traj'), html=os.path.join(options.outdir, "%s_traj.html" % confid), rd=rd, job_id='report_traj-write') if options.compmake_command is not None: compmake.batch_command(options.compmake_command) else: compmake_console()
def main(): parser = OptionParser(usage=description) parser.add_option("--flydra_db", default="saccade_data_flydradb", help="Main data directory") parser.add_option( "--interactive", action="store_true", default=False, help="Starts an interactive compmake session." ) parser.add_option("--report", default="saccade_report", help="Saccade report directory") parser.add_option("--groups", default=None, help="Which groups to consider") parser.add_option("--configurations", default=None, help="Which configurations to consider") parser.add_option("--combid", default=None, help="How to name this combination of groups/configs.") (options, args) = parser.parse_args() # @UnusedVariable if args: raise Exception("Spurious arguments %r." % args) db = FlydraDB(options.flydra_db) robust_split = lambda s: filter(lambda x: x, s.split(",")) if not options.groups in [None, "all"]: groups = robust_split(options.groups) if not groups: raise Exception("No groups specified.") groupset = "_".join(groups) else: groups = db.list_groups() groupset = "all" if not groups: raise Exception("No groups found.") if not options.configurations in [None, "all"]: configurations = robust_split(options.configurations) if not configurations: raise Exception("No configuration specified") confset = "_".join(configurations) else: configurations = db.list_all_versions_for_table(SACCADES_TABLE) confset = "all" configurations = set() for group in groups: configurations.update(db.list_versions_for_table_in_group(group, SACCADES_TABLE)) configurations = natsorted(configurations) if not configurations: raise Exception("No valid versions of table %r found." % SACCADES_TABLE) print ("I will consider the configurations: %r" % configurations) if options.combid is None: combination = "%s_%s" % (groupset, confset) else: combination = options.combid print ("I call this combination %r." % combination) output_dir = os.path.join(options.report, combination) if not os.path.exists(output_dir): os.makedirs(output_dir) set_namespace("master_plot_%s" % combination) # we maintain several indices # key = (group, configuration, plot) index_group_plots = {} # key = (sample, plot) index_sample_expdata_plots = {} # key = (sample, configuration, plot) index_sample_saccades_plots = {} # First we index the DB print ("Looking for data in database...") all_samples = set() group2samples = {} configurations_for_group = {} group_has_exp_data = {} for group in groups: group2samples[group] = db.list_samples_for_group(group) all_samples.update(group2samples[group]) if not group2samples[group]: raise Exception("Empty group %r." % group) available = db.list_versions_for_table_in_group(group=group, table=SACCADES_TABLE) configurations_for_group[group] = set(configurations).intersection(available) if not configurations_for_group[group]: print ("No configurations found for %r; available %r" % (group, available)) group_has_exp_data[group] = db.group_has_table(group, EXP_DATA_TABLE) all_samples = natsorted(all_samples) # print info print ("Summary:") for group in groups: msg = " group {group:>20} samples: {nsamples:3} " " configurations: {nconf:3} raw data? {data}".format( group=group, nsamples=len(group2samples[group]), nconf=len(configurations_for_group[group]), data=group_has_exp_data[group], ) print (msg) # TODO: iterate by sample, not by group for group in groups: for configuration in configurations_for_group[group]: for plot in group_plots: job_id = "%s-%s-%s" % (group, configuration, plot.id) index_group_plots[(group, configuration, plot.id)] = comp( wrap_group_plot, options.flydra_db, group, configuration, plot.command, plot.args, job_id=job_id ) for sample, plot in prod(group2samples[group], sample_saccades_plots): job_id = "%s-%s-%s" % (sample, configuration, plot.id) key = (sample, configuration, plot.id) if key in index_sample_saccades_plots: # we already did it as part of another group continue index_sample_saccades_plots[key] = comp( wrap_sample_saccades_plot, options.flydra_db, sample, configuration, plot.command, plot.args, job_id=job_id, ) if group_has_exp_data[group]: for sample, plot in prod(group2samples[group], sample_expdata_plots): job_id = "%s-%s" % (sample, plot.id) key = (sample, plot.id) if key in index_sample_expdata_plots: # we already did it as part of another group continue index_sample_expdata_plots[key] = comp( wrap_sample_expdata_plot, options.flydra_db, sample, plot.command, plot.args, job_id=job_id ) # now we create the indices # fix configuration, function; iterate groups for configuration, plot in itertools.product(configurations, group_plots): subs = [] descs = [] page_id = "%s.%s" % (configuration, plot.id) for group, group_desc in order_groups(groups): if not configuration in configurations_for_group[group]: continue descs.append(group_desc) subs.append(index_group_plots[(group, configuration, plot.id)]) if not subs: raise Exception("no groups for configuration %r." % configuration) job_id = page_id comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id) comp( create_gui, filename=os.path.join(output_dir, "group_plots.html"), menus=[ ("Detector", configurations, configurations), ("Plot/table", map(lambda x: x.id, group_plots), map(lambda x: x.description, group_plots)), ], job_id="gui-group_plots", ) # fix group, function; iterate samples for group in groups: if not group_has_exp_data[group]: continue for plot in sample_expdata_plots: subs = [] descs = [] for sample in group2samples[group]: descs.append(sample) subs.append(index_sample_expdata_plots[(sample, plot.id)]) page_id = "%s.%s" % (group, plot.id) job_id = page_id comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id) # get the ordered group lists and desc ordered_groups = map(lambda t: t[0], order_groups(groups)) ordered_groups_desc = map(lambda t: t[1], order_groups(groups)) comp( create_gui, filename=os.path.join(output_dir, "expdata_plots.html"), menus=[ ("Group", ordered_groups, ordered_groups_desc), ( "Plot/table", map(lambda x: x.id, sample_expdata_plots), map(lambda x: x.description, sample_expdata_plots), ), ], job_id="gui-expdata_plots", ) # fix configuration, group, function; iterate samples for group in groups: for configuration in configurations: if not configuration in configurations_for_group[group]: for plot in sample_saccades_plots: page_id = "%s.%s.%s" % (configuration, group, plot.id) comp( write_empty, page_id, output_dir, "Group %r has not been processed with algorithm %r." % (group, configuration), job_id=page_id, ) continue for plot in sample_saccades_plots: subs = [] descs = [] for sample in group2samples[group]: descs.append(sample) r = index_sample_saccades_plots[(sample, configuration, plot.id)] subs.append(r) page_id = "%s.%s.%s" % (configuration, group, plot.id) job_id = page_id comp(combine_reports, subs, descs, page_id, output_dir, job_id=job_id) comp( create_gui, filename=os.path.join(output_dir, "saccade_plots.html"), menus=[ ("Detector", configurations, configurations), ("Group", ordered_groups, ordered_groups_desc), ( "Plot/table", map(lambda x: x.id, sample_saccades_plots), map(lambda x: x.description, sample_saccades_plots), ), ], job_id="gui-saccade_plots", ) # fix configuration, sample; plot fullsscreen for group in groups: for configuration in configurations: for sample in group2samples[group]: # XXX make it clenaer if not configuration in configurations_for_group[group]: for plot in sample_fullscreen_plots: page_id = "%s.%s.%s" % (sample, configuration, plot.id) comp( write_empty, page_id, output_dir, 'Group %s has not been processed with algorithm "%s".' % (group, configuration), job_id=page_id, ) # print "skipping sample %s group %s config %s" %\ # (sample,group, configuration) continue if not group_has_exp_data[group]: for plot in sample_fullscreen_plots: page_id = "%s.%s.%s" % (sample, configuration, plot.id) comp( write_empty, page_id, output_dir, "Group %r does not have raw experimental data." % (group), job_id=page_id, ) continue for plot in sample_fullscreen_plots: job_id = "%s-%s-%s" % (sample, configuration, plot.id) # FIXME: error if sample in 2 groups job = comp( wrap_sample_saccades_plot, options.flydra_db, sample, configuration, plot.command, plot.args, job_id=job_id, ) page_id = "%s.%s.%s" % (sample, configuration, plot.id) comp(write_report, job, output_dir, page_id, job_id=job_id + "-write_report") comp( create_gui, filename=os.path.join(output_dir, "sample_fullscreen_plots.html"), menus=[ ("Sample", all_samples, all_samples), ("Detector", configurations, configurations), ( "Plot/table", map(lambda x: x.id, sample_fullscreen_plots), map(lambda x: x.description, sample_fullscreen_plots), ), ], job_id="gui-sample_fullscreen_plots", ) tabs = [ ( "group_plots", "By group", "This set displays one plot/table for each group of samples. " "You have the further choice of detection algorithm and plot/table to display.", ), ( "saccade_plots", "By sample", "This set displays one plot/table for each individual sample. " "You have the further choice of which group to consider, which " "detection algorithm, and which plot/table to display.", ), ( "expdata_plots", "By sample (raw)", "This set displays one plot/table for each individual sample, " " produced from the raw data (no saccade detection, so no choice of detector). " "You have the further choice of which group to consider, " "and which plot/table to display." " Note that some samples might be missing; for example, we don't use " " raw orientation data for the Mamarama samples.", ), ( "sample_fullscreen_plots", "By sample, single page", "This set displays one entire page for each sample. " "You have the further choice of sample, " "detection algorithm, and which plot/table to display.", ), ] comp(create_main_gui, tabs, filename=os.path.join(output_dir, "main.html"), job_id="gui-main") if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command("make all") # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list("todo")) if todo: print ("Still %d jobs to do." % len(todo)) sys.exit(-2)
def go(self): # check that if we have a parent who is a quickapp, # then use its context qapp_parent = self.get_qapp_parent() if qapp_parent is not None: # self.info('Found parent: %s' % qapp_parent) context = qapp_parent.child_context self.define_jobs_context(context) return else: # self.info('Parent not found') pass if False: import resource gbs = 5 max_mem = long(gbs * 1000 * 1048576L) resource.setrlimit(resource.RLIMIT_AS, (max_mem, -1)) resource.setrlimit(resource.RLIMIT_DATA, (max_mem, -1)) options = self.get_options() if self.get_qapp_parent() is None: # only do this if somebody didn't do it before if not options.contracts: msg = 'PyContracts disabled for speed. Use --contracts to activate.' self.logger.warning(msg) contracts.disable_all() warnings.warn('removed configuration below') # (start) output_dir = options.output # Compmake storage for results storage = os.path.join(output_dir, 'compmake') sf = StorageFilesystem(storage, compress=True) # sf = StorageFilesystem2(directory) # sf = MemoryCache(sf) set_compmake_db(sf) # use_filesystem(storage) read_rc_files() context = CompmakeContext(parent=None, qapp=self, job_prefix=None, output_dir=output_dir) self.context = context original = get_comp_prefix() self.define_jobs_context(context) comp_prefix(original) context.finalize_jobs() if context.n_comp_invocations == 0: # self.comp was never called msg = 'No jobs defined.' raise ValueError(msg) else: if not options.console: batch_result = batch_command(options.command) if isinstance(batch_result, str): ret = QUICKAPP_COMPUTATION_ERROR elif isinstance(batch_result, int): if batch_result == 0: ret = 0 else: # xxx: discarded information ret = QUICKAPP_COMPUTATION_ERROR else: assert False return ret else: compmake_console() return 0
def test_order(): from compmake import comp, batch_command # make A -> B(fail) -> C for i in range(10): comp(job_failure, job_id='F%d' % i) batch_command('parmake n=2')
def main(): parser = OptionParser() group = OptionGroup(parser, "Files and directories") group.add_option("--outdir", help='Directory with variables.pickle and where ' 'the output will be placed.') parser.add_option_group(group) group = OptionGroup(parser, "Experiments options") group.add_option("--fast", default=False, action='store_true', help='Disables sanity checks.') group.add_option("--set", default='*', help='[= %default] Which combinations to run.') group.add_option("--seed", default=None, type='int', help='[= %default] Seed for random number generator.') parser.add_option_group(group) group = OptionGroup(parser, "Compmake options") group.add_option("--remake", default=False, action='store_true', help='Remakes all (non interactive).') group.add_option("--report", default=False, action='store_true', help='Cleans and redoes all reports (non interactive).') group.add_option("--report_stats", default=False, action='store_true', help='Cleans and redoes the reports for the stats. (non interactive)') parser.add_option_group(group) (options, args) = parser.parse_args() #@UnusedVariable np.random.seed(options.seed) if options.fast: disable_all() assert not args assert options.outdir is not None available_algorithms, available_test_cases, available_sets = get_everything() which = expand_string(options.set, list(available_sets.keys())) if len(which) == 1: compmake_storage = os.path.join(options.outdir, 'compmake', which[0]) else: compmake_storage = os.path.join(options.outdir, 'compmake', 'common_storage') use_filesystem(compmake_storage) print('Staging creation of test cases reports') test_cases = {} test_case_reports = {} def stage_test_case_report(tcid): if not tcid in available_test_cases: msg = ('Could not find test case %r \n %s' % (tcid, available_test_cases.keys())) raise Exception(msg) if not tcid in test_cases: command, args = available_test_cases[tcid] job_id = 'test_case_data-%s' % tcid test_cases[tcid] = comp(command, job_id=job_id, **args) if not tcid in test_case_reports: job_id = 'test_case-%s-report' % tcid report = comp(create_report_tc, tcid, test_cases[tcid], job_id=job_id) job_id += '-write' filename = os.path.join(options.outdir, 'test_cases', '%s.html' % tcid) comp(write_report, report, filename, job_id=job_id) test_case_reports[tcid] = report return test_case_reports[tcid] # set of tuple (algo, test_case) executions = {} def stage_execution(tcid, algid): stage_test_case_report(tcid) key = (tcid, algid) if not key in executions: test_case = test_cases[tcid] algo_class, algo_params = available_algorithms[algid] job_id = 'solve-%s-%s-run' % (tcid, algid) results = comp(run_combination, tcid, test_case, algo_class, algo_params, job_id=job_id) executions[key] = results exc_id = '%s-%s' % (tcid, algid) # Create iterations report job_id = 'solve-%s-report' % exc_id report = comp(create_report_execution, exc_id, tcid, test_case, algo_class, algo_params, results, job_id=job_id) job_id += '-write' filename = os.path.join(options.outdir, 'executions', '%s-%s.html' % (tcid, algid)) comp(write_report, report, filename, job_id=job_id) return executions[key] for comb_id in which: comb = available_sets[comb_id] alg_ids = expand_string(comb.algorithms, available_algorithms.keys()) tc_ids = expand_string(comb.test_cases, available_test_cases.keys()) print('Set %r has %d test cases and %d algorithms (~%d jobs in total).' % (comb_id, len(alg_ids), len(tc_ids), len(alg_ids) * len(tc_ids) * 2)) deps = {} for t, a in itertools.product(tc_ids, alg_ids): deps[(t, a)] = stage_execution(t, a) job_id = 'tex-%s' % comb_id comp(create_tables_for_paper, comb_id, tc_ids, alg_ids, deps, job_id=job_id) job_id = 'set-%s-report' % comb_id report = comp(create_report_comb_stats, comb_id, tc_ids, alg_ids, deps, job_id=job_id) job_id += '-write' filename = os.path.join(options.outdir, 'stats', '%s.html' % comb_id) comp(write_report, report, filename, job_id=job_id) if options.report or options.report_stats: if options.report: batch_command('clean *-report*') elif options.report_stats: batch_command('clean set-* tex*') batch_command('parmake') elif options.remake: batch_command('clean *') batch_command('make set-* tex-*') else: compmake_console()
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--image", help="Which image to plot.") parser.add_option("--filter", help="Which procgraph filter to use to plot.", default="flydra_simple_video_filter") parser.add_option("--interactive", help="Start compmake interactive session." " Otherwise run in batch mode", default=False, action="store_true") (options, args) = parser.parse_args() if options.image is None: print "Usage: %s [--db DB] --image <image> [ids]" % sys.argv[0] sys.exit(-1) db = FlydraDB(options.db) set_namespace('video_image_%s' % options.image) if args: samples = args else: # look for samples with the rows table samples = db.list_samples() samples = filter(lambda x: db.has_table(x, options.image), samples) if not samples: raise Exception('No samples found at all with available image "%s".' % \ options.image) for id in samples: if not db.has_table(id, options.image): raise Exception('Sample %s does not have table "%s".' % (id, options.image)) config = {'sample': id, 'db': options.db, 'image': options.image, 'filter': options.filter} comp(pg, 'flydra_simple_video', config, job_id="%s" % id) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: print('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def main(): parser = OptionParser(usage=description) parser.add_option( "--db", default='flydra_db_directory', help="FlydraDB directory") parser.add_option("--model", help="ProcGraph model name.") parser.add_option( "--needs", help="Comma-separated list of tables required", default="rows,luminance") parser.add_option( "--interactive", help="Start compmake interactive session." " Otherwise run in batch mode", default=False, action="store_true") (options, args) = parser.parse_args() if options.model is None: print "Please specify the model." sys.exit(-3) print("Using FlydraDB directory %r." % options.db) db = FlydraDB(options.db, False) # TODO: make the storage inside options.db? set_namespace('run_pg_model_%s' % options.model) tables = options.needs.split(',') if args: samples = args for sample in samples: if not db.has_sample(sample): raise Exception('Unknown sample %r' % sample) else: samples = db.list_samples() if not samples: print 'No samples found' num_ok = 0 for id in samples: enough = all(map(lambda t: db.has_table(id, t), tables)) if not enough: continue num_ok += 1 config = {'sample': id, 'db': options.db} comp(pg, options.model, config, job_id=id) logger.info( "Found %d/%d samples with tables %s." % (num_ok, len(samples), tables)) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def rlearn(config, parser): #@UnusedVariable t0 = time.time() """ Displays the learned DDS """ parser.add_option("-n", "--nthreads", help="Number of threads", type='int', default='4') parser.add_option("-r", "--nrefine", help="Number of time to refine learning", type='int', default='2') parser.add_option("-s", "--streams", help="Which streams to use.", default="*") parser.add_option("-i", "--comb", default="default") parser.add_option("-l", "--learners", help="Learner config.", default="*") parser.add_option("-o", "--output", help="Output directory", default='out/dp-rlearn/') parser.add_option( "--sensels", default=None, help="Sensel indices to use for debuging refining module") parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") parser.add_option("--show", default=None, help="Name of learners to report") options = parser.parse_options() if options.show is not None: diffeomorphism2d_continuous.make_report(options.show.split(',')) sys.exit() nthreads = options.nthreads nrefine = options.nrefine learners = config.learners.expand_names(options.learners) streams = config.streams.expand_names(options.streams) if len(learners) > 1: logger.warn('Multiple learners are not supported for now') if len(streams) > 1: logger.warn('Multiple streams are not supported for now') id_comb = ",".join(streams) + "-" + ",".join(learners) outdir = os.path.join(options.output, id_comb) storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() rm = ReportManager(os.path.join(outdir, 'reports')) jobs_rlearn(config, rm, learners, streams, outdir, nthreads, nrefine, options.sensels) # Time and report the learning comp(learning_times_rlearn, outdir, learners, streams, nthreads, nrefine) rm.create_index_job() if options.command: return batch_command(options.command) else: compmake_console() return 0 logger.info("Done after time: " + str(time.time() - t0) + ' seconds')
result = comp(random_motion_simulation, world_gen=my_world_gen, vehicle=vehicle, random_pose_gen=random_pose_gen, num_iterations=num_iterations, random_commands_gen=random_commands_gen, dt=dt, processing_class=AffineModel, job_id='affine') vehicle_report = comp(create_report_affine, state=result, report_id=vname) all_vehicle_report.append(vehicle_report) comp_prefix() def create_report(id, children): return Node(id=id, children=children) affine_report = comp(create_report, id='affine', children=all_vehicle_report) def write_report(report, basename): # report.to_latex_document(basename + '.tex') node_to_html_document(report, basename + '.html') comp(write_report, affine_report, "reports/affine") batch_command('make all')
def add_and_execute(self, function): from compmake import comp, batch_command comp(function) batch_command('clean') return batch_command('make')
# we now add an exception if param2 == 11: raise Exception('11 is your unlucky number.') print('func2(%s, %s)' % (res1, param2)) time.sleep(1) return res1 + param1 def draw(res2): print('draw(%s)' % res2) print('Defining jobs...') for param1 in [1,2,3]: for param2 in [10,11,12]: res1 = comp(func1, param1) res2 = comp(func2, res1, param2) comp(draw, res2) print('Ready to run...') import compmake if compmake.is_inside_compmake_script(): print('Detected that we were imported by compmake.') else: interactive = True if interactive: print('Presenting an interactive console') compmake.compmake_console() else: print('Running the computation in batch mode') compmake.batch_command('parmake n=4')
def main(): parser = OptionParser() group = OptionGroup(parser, "Files and directories") group.add_option("--outdir", help='Directory with variables.pickle and where ' 'the output will be placed.') parser.add_option_group(group) group = OptionGroup(parser, "Experiments options") group.add_option("--fast", default=False, action='store_true', help='Disables sanity checks.') group.add_option("--set", default='*', help='[= %default] Which combinations to run.') group.add_option("--seed", default=None, type='int', help='[= %default] Seed for random number generator.') parser.add_option_group(group) group = OptionGroup(parser, "Compmake options") group.add_option("--remake", default=False, action='store_true', help='Remakes all (non interactive).') group.add_option("--report", default=False, action='store_true', help='Cleans and redoes all reports (non interactive).') group.add_option( "--report_stats", default=False, action='store_true', help='Cleans and redoes the reports for the stats. (non interactive)') parser.add_option_group(group) (options, args) = parser.parse_args() #@UnusedVariable np.random.seed(options.seed) if options.fast: disable_all() assert not args assert options.outdir is not None available_algorithms, available_test_cases, available_sets = get_everything( ) which = expand_string(options.set, list(available_sets.keys())) if len(which) == 1: compmake_storage = os.path.join(options.outdir, 'compmake', which[0]) else: compmake_storage = os.path.join(options.outdir, 'compmake', 'common_storage') use_filesystem(compmake_storage) print('Staging creation of test cases reports') test_cases = {} test_case_reports = {} def stage_test_case_report(tcid): if not tcid in available_test_cases: msg = ('Could not find test case %r \n %s' % (tcid, available_test_cases.keys())) raise Exception(msg) if not tcid in test_cases: command, args = available_test_cases[tcid] job_id = 'test_case_data-%s' % tcid test_cases[tcid] = comp(command, job_id=job_id, **args) if not tcid in test_case_reports: job_id = 'test_case-%s-report' % tcid report = comp(create_report_tc, tcid, test_cases[tcid], job_id=job_id) job_id += '-write' filename = os.path.join(options.outdir, 'test_cases', '%s.html' % tcid) comp(write_report, report, filename, job_id=job_id) test_case_reports[tcid] = report return test_case_reports[tcid] # set of tuple (algo, test_case) executions = {} def stage_execution(tcid, algid): stage_test_case_report(tcid) key = (tcid, algid) if not key in executions: test_case = test_cases[tcid] algo_class, algo_params = available_algorithms[algid] job_id = 'solve-%s-%s-run' % (tcid, algid) results = comp(run_combination, tcid, test_case, algo_class, algo_params, job_id=job_id) executions[key] = results exc_id = '%s-%s' % (tcid, algid) # Create iterations report job_id = 'solve-%s-report' % exc_id report = comp(create_report_execution, exc_id, tcid, test_case, algo_class, algo_params, results, job_id=job_id) job_id += '-write' filename = os.path.join(options.outdir, 'executions', '%s-%s.html' % (tcid, algid)) comp(write_report, report, filename, job_id=job_id) return executions[key] for comb_id in which: comb = available_sets[comb_id] alg_ids = expand_string(comb.algorithms, available_algorithms.keys()) tc_ids = expand_string(comb.test_cases, available_test_cases.keys()) print( 'Set %r has %d test cases and %d algorithms (~%d jobs in total).' % (comb_id, len(alg_ids), len(tc_ids), len(alg_ids) * len(tc_ids) * 2)) deps = {} for t, a in itertools.product(tc_ids, alg_ids): deps[(t, a)] = stage_execution(t, a) job_id = 'tex-%s' % comb_id comp(create_tables_for_paper, comb_id, tc_ids, alg_ids, deps, job_id=job_id) job_id = 'set-%s-report' % comb_id report = comp(create_report_comb_stats, comb_id, tc_ids, alg_ids, deps, job_id=job_id) job_id += '-write' filename = os.path.join(options.outdir, 'stats', '%s.html' % comb_id) comp(write_report, report, filename, job_id=job_id) if options.report or options.report_stats: if options.report: batch_command('clean *-report*') elif options.report_stats: batch_command('clean set-* tex*') batch_command('parmake') elif options.remake: batch_command('clean *') batch_command('make set-* tex-*') else: compmake_console()
def pixlearn(config, parser): parser.add_option("-n", "--nthreads", help="Number of threads", type='int', default='4') parser.add_option("--distribute", type='str', default='random', help="Strategy to distribute sensles to threads") parser.add_option("-s", "--id_stream", help="Which streams to use.", default="*") parser.add_option("-o", "--output", help="Output directory", default='out/dp-pixlearn/') parser.add_option("-l", "--id_learner", help="Learner config.", default="*") parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") options = parser.parse_options() nthreads = options.nthreads id_learner = config.learners.expand_names(options.id_learner) id_stream = config.streams.expand_names(options.id_stream) id_comb = ",".join(id_stream) + "-" + ",".join(id_learner) outdir = os.path.join(options.output, id_comb) storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() rm = ReportManager(os.path.join(outdir, 'reports')) pdb.set_trace() if options.distribute == 'random': max_nsensels = 40 * 30 # Generate a list with range of all indexes and assign them to threads all_indicies = np.array(range(max_nsensels)) dist = np.random.randint(nthreads, size=max_nsensels) sensel_indexes = [] for i in range(nthreads): sensel_indexes.append(all_indicies[dist == i]) if options.distribute == 'demo4': sensel_indexes = [[0, 1, 2, 3, 40, 41, 42, 43, 80, 81, 82, 83, 120, 121, 122, 123], [340, 341, 342, 343, 380, 381, 382, 383, 420, 421, 422, 423, 460, 461, 462, 463], [650, 651, 652, 653, 690, 691, 692, 693, 730, 731, 732, 733, 770, 771, 772, 773], [1076, 1077, 1078, 1079, 1116, 1117, 1118, 1119, 1156, 1157, 1158, 1159, 1196, 1197, 1198, 1199]] if len(id_learner) > 1 or len(id_stream) > 1: logger.warning('learners and streams after index 0 will be ignored') id_learner = id_learner[0] id_stream = id_stream[0] commands = [[256, 0, 0], [-256, 0, 0]] states = [[100]] # # Parallel part of code # Initiate parallel learning group = [] for i in range(nthreads): estimator_i = comp(sensel_group_learn, config, id_learner, id_stream, sensel_indexes[i]) group.append(estimator_i) diffeo_system_i = comp(estimator_summarize, estimator_i, commands, states, 'diffeo' + str(i)) estimator_report = comp(report_estimator, 'estimator' + str(i), estimator_i) rm.add(estimator_report, 'estimator' + str(i), id_learner=id_learner, id_stream=id_stream) diffeo_report = comp(report_dds, 'diffeo' + str(i), diffeo_system_i) rm.add(diffeo_report, 'diffeo' + str(i), id_learner=id_learner, id_stream=id_stream) estimator_main = comp(join_estimators, group) main_system = comp(estimator_summarize, estimator_main, commands, states, 'diffeo' + str(i)) diffeo_report = comp(report_dds, 'dds-%s-%s' % (id_stream, id_learner), main_system, job_id='learn-%s-%s-summarize-report' % (id_stream, id_learner)) rm.add(comp(empty_report), 'empty') rm.add(diffeo_report, 'dds', id_learner=id_learner, id_stream=id_stream) rm.create_index_job() if options.command: # return batch_command(options.command) batch_command(options.command) else: compmake_console()
def batch_process_manager(data_central, which_sets, command=None): try: import compmake # @UnusedImport except: logger.error('Compmake not installed; multiprocessor ' 'processes not available.') raise from compmake import (comp_prefix, use_filesystem, compmake_console, batch_command) batch_config = BatchConfigMaster() configs = data_central.get_dir_structure().get_config_directories() for config in configs: batch_config.load(config) sets_available = batch_config.sets.keys() # logger.info('Available: %r' % sets_available) # logger.info('Sets: %r' % which_sets) which_sets_int = expand_string(which_sets, options=sets_available) if not which_sets_int: msg = 'Specified sets %r not found.' % which_sets msg += ' Available: %s' % sets_available raise UserError(msg) # logger.info('Expanded: %r' % which_sets) for x in which_sets_int: if not x in sets_available: msg = 'Set %r not available.' % x raise UserError(msg) if len(which_sets_int) == 1: combid = which_sets[0] else: combid = '-'.join(which_sets) # Create the new root root = data_central.root root_set = os.path.join(data_central.root, 'sets', combid) safe_makedirs(root_set) data_central_set = DataCentral(root_set) # add symbolic links to logs and config main_config = os.path.realpath(os.path.join(root, 'config')) set_config = os.path.join(root_set, 'config') safe_symlink(main_config, set_config) safe_makedirs(os.path.join(root_set, 'logs')) safe_symlink(os.path.join(root, 'logs'), os.path.join(root_set, 'logs', 'original')) storage = data_central_set.get_dir_structure().get_storage_dir() compmake_storage = os.path.join(storage, 'compmake') logger.debug('Using storage directory %r.' % friendly_path(compmake_storage)) use_filesystem(compmake_storage) for id_set in which_sets: if len(which_sets) > 1: comp_prefix(id_set) try: spec = batch_config.sets[x] batch_set(data_central_set, id_set, spec) except ConfToolsException: msg = ('Bad configuration for the set %r with spec\n %s' % (id_set, pformat(spec))) logger.error(msg) raise if command: return batch_command(command) else: compmake_console() return 0
def main(): parser = OptionParser(usage=description) parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--interactive", default=False, action="store_true", help="Start a compmake interactive session." " Otherwise run in batch mode") parser.add_option("--empty_group_ok", default=False, action="store_true", help="do not give up if one group does not have samples ") (options, args) = parser.parse_args() #@UnusedVariable if options.db is None: logger.error('Please specify a directory using --db.') sys.exit(-1) outdir = os.path.join(options.db, 'out/saccade_view_joint_analysis') db = FlydraDB(options.db, False) set_namespace('saccade_view_joint_analysis') # for each image we do a different report data = {} for image in images: # For each image we have different tables tables = ["saccades_view_%s_%s" % (view.id, image.id) for view in views] all_available = [x for x in db.list_samples() if db.has_saccades(x) and all([db.has_table(x, table) for table in tables])] # We further divide these in post and nopost groups_samples = { 'posts': filter(lambda s: db.get_attr(s, 'stimulus') != 'nopost', all_available), 'noposts': filter(lambda s: db.get_attr(s, 'stimulus') == 'nopost', all_available) } # now, for each group for group in groups: is_hallucination = image.id.startswith('h') white_arena = image.id.endswith('_w') if (not is_hallucination) and white_arena and (group.id == 'noposts'): # if there are not posts, it's useless continue samples = groups_samples[group.id] if not samples: print "Warning: no samples for %s/%s" % (image.id, group.id) continue # global statistics key = (group.id, image.id) job_id = "%s-%s" % key data[key] = comp(compute_stats, options.db, samples, image.id, job_id=job_id) for saccades_set, direction in prod(saccades_sets, dirs): view2result = {} for i, view in enumerate(views): table = tables[i] key = Exp(image=image.id, group=group.id, view=view.id, dir=direction.id, saccades_set=saccades_set.id) job_id = "%s-%s-%s-%s-%s" % key result = comp(compute_saccade_stats, options.db, samples, table, [direction.args, saccades_set.args], job_id=job_id) data[key] = result view2result[view.id] = result page_id = make_page_id(image=image.id, group=group.id, dir=direction.id, saccades_set=saccades_set.id) comp(render_page, view2result, outdir, page_id, job_id=page_id) for saccades_set in saccades_sets: table = "saccades_view_start_%s" % (image.id) exp_id = '%s_%s_%s' % (image.id, group.id, saccades_set.id) results = comp(bet_on_flies, options.db, samples, table, saccades_set, job_id='lasvegas-' + exp_id + '-bet') page_id = exp_id comp(las_vegas_report, os.path.join(outdir, 'lasvegas'), page_id, results, job_id='lasvegas-' + exp_id + '-report') db.close() comp(add_comparisons, data, outdir) filename = os.path.join(outdir, 'gui.html') comp(create_gui_new, filename, menus) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # exit with error if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)