def main(): set_namespace('env_stats') parser = OptionParser() parser.add_option("--db", default='flydra_db', help="FlydraDB directory") (options, args) = parser.parse_args() #@UnusedVariable db = FlydraDB(options.db, False) outdir = os.path.join(options.db, 'out/environment_stats') images = ["luminance", "contrast", "luminance_w", "contrast_w", "hluminance_w", "hcontrast_w"] for image in images: samples = [x for x in db.list_samples() if db.get_attr(x, 'stimulus', None) != 'nopost' and db.has_table(x, image)] if not samples: print "No samples for %s" % samples continue comp_prefix(image) data = comp(compute_environment_autocorrelation, options.db, samples, image) comp(create_report, data, image, outdir) db.close() compmake_console()
def main(): parser = OptionParser() parser.add_option("--db", default='flydra_db', help="Data directory") parser.add_option("--image", default="luminance", help="Rendered image to use -- " " corresponding to image 'saccades_view_{start,stop}_X'") parser.add_option("--interactive", help="Start an interactive compmake session." " Otherwise run in batch mode. ", default=False, action="store_true") (options, args) = parser.parse_args() #@UnusedVariable if options.db is None: logger.error('Please specify a directory using --db.') sys.exit(-1) view_start = 'saccades_view_start_%s' % options.image view_stop = 'saccades_view_stop_%s' % options.image view_rstop = 'saccades_view_rstop_%s' % options.image db = FlydraDB(options.db, False) # all samples with enough data all_available = lambda x: db.has_saccades(x) and \ db.has_table(x, view_start) and \ db.has_table(x, view_stop) and \ db.has_table(x, view_rstop) samples = filter(all_available, db.list_samples()) set_namespace('saccade_view_show_%s' % options.image) for sample in samples: comp_prefix(sample) comp(create_and_write_report, options.db, sample, options.image) if options.interactive: # start interactive session compmake_console() else: # batch mode # try to do everything batch_command('make all') # start the console if we are not done # (that is, make all failed for some reason) todo = list(parse_job_list('todo')) if todo: logger.info('Still %d jobs to do.' % len(todo)) sys.exit(-2)
def comp(self, f, *args, **kwargs): """ Simple wrapper for Compmake's comp function. Use this instead of "comp". """ self.count_comp_invocations() comp_prefix(self._job_prefix) extra_dep = self._extra_dep + kwargs.get('extra_dep', []) kwargs['extra_dep'] = extra_dep promise = comp(f, *args, **kwargs) self._jobs[promise.job_id] = promise return promise
def dp_batch_main(config, parser): """ Runs batch planning experiments from batch configuration files. """ parser.add_option("-o", "--output", default='out/dp-batch', help="Output directory") parser.add_option("-c", "--command", help="Command to pass to compmake for batch mode") options, which = parser.parse() if not which: todo = config.sets.keys() id_comb = 'all' else: todo = config.sets.expand_names(which) id_comb = "+".join(sorted(todo)) logger.info('Batch sets to do: %s' % todo) outdir = os.path.join(options.output, 'set-%s' % id_comb) # Compmake storage for results storage = os.path.join(outdir, 'compmake') use_filesystem(storage) read_rc_files() for id_set in todo: logger.info('Instantiating batch set %s' % id_set) spec = config.sets[id_set] try: algos = config.algos.expand_names(spec['algorithms']) testcases = config.testcases.expand_names(spec['testcases']) comp_prefix('%s' % id_set) b_outdir = os.path.join(outdir, id_set) create_bench_jobs(config=config, algos=algos, testcases=testcases, outdir=b_outdir) except: logger.error('Error while instantiating batch\n%s' % pformat(spec)) raise if options.command: return batch_command(options.command) else: compmake_console() return 0
# val = (random.rand(1) - 0.5) * 2 # i = int(floor(random.rand(1) * 2.99)) # u = zeros(3) # u[i] = val # return u random_commands_gen = my_random_commands_gen num_iterations = 500 dt = 0.1 vehicle_list = vehicles_list_A() all_vehicle_report = [] for vname, vehicle in vehicle_list: vname_tex = vname.replace('_', '-') comp_prefix(vname) result = comp(random_motion_simulation, world_gen=my_world_gen, vehicle=vehicle, random_pose_gen=random_pose_gen, num_iterations=num_iterations, random_commands_gen=random_commands_gen, dt=dt, processing_class=AffineModel, job_id='affine') vehicle_report = comp(create_report_affine, state=result, report_id=vname) all_vehicle_report.append(vehicle_report) comp_prefix() def create_report(id, children):
def go(self): # check that if we have a parent who is a quickapp, # then use its context qapp_parent = self.get_qapp_parent() if qapp_parent is not None: # self.info('Found parent: %s' % qapp_parent) context = qapp_parent.child_context self.define_jobs_context(context) return else: # self.info('Parent not found') pass if False: import resource gbs = 5 max_mem = long(gbs * 1000 * 1048576L) resource.setrlimit(resource.RLIMIT_AS, (max_mem, -1)) resource.setrlimit(resource.RLIMIT_DATA, (max_mem, -1)) options = self.get_options() if self.get_qapp_parent() is None: # only do this if somebody didn't do it before if not options.contracts: msg = 'PyContracts disabled for speed. Use --contracts to activate.' self.logger.warning(msg) contracts.disable_all() warnings.warn('removed configuration below') # (start) output_dir = options.output # Compmake storage for results storage = os.path.join(output_dir, 'compmake') sf = StorageFilesystem(storage, compress=True) # sf = StorageFilesystem2(directory) # sf = MemoryCache(sf) set_compmake_db(sf) # use_filesystem(storage) read_rc_files() context = CompmakeContext(parent=None, qapp=self, job_prefix=None, output_dir=output_dir) self.context = context original = get_comp_prefix() self.define_jobs_context(context) comp_prefix(original) context.finalize_jobs() if context.n_comp_invocations == 0: # self.comp was never called msg = 'No jobs defined.' raise ValueError(msg) else: if not options.console: batch_result = batch_command(options.command) if isinstance(batch_result, str): ret = QUICKAPP_COMPUTATION_ERROR elif isinstance(batch_result, int): if batch_result == 0: ret = 0 else: # xxx: discarded information ret = QUICKAPP_COMPUTATION_ERROR else: assert False return ret else: compmake_console() return 0
def generate_saccade(model, num): sequence = model.sample_saccade_sequence(num) return saccades_to_ndarray(sequence) def create_report(id, children): return ReportNode(id=id, children=children) def write_report(report, basename): report.to_latex_document(basename + '.tex') report.to_html_document(basename + '.html') from compmake import comp, comp_prefix #@UnresolvedImport models = create_models() reports = [] for model_name, model in models: comp_prefix(model_name) saccades = comp(generate_saccade, model, num=10000) report = comp(analyze_saccades, report_id=model_name, saccades=saccades) reports.append(report) comp_prefix() report = comp(create_report, id='all_models', children=reports) comp(write_report, report, 'all_models')
def batch_process_manager(data_central, which_sets, command=None): try: import compmake # @UnusedImport except: logger.error('Compmake not installed; multiprocessor ' 'processes not available.') raise from compmake import (comp_prefix, use_filesystem, compmake_console, batch_command) batch_config = BatchConfigMaster() configs = data_central.get_dir_structure().get_config_directories() for config in configs: batch_config.load(config) sets_available = batch_config.sets.keys() # logger.info('Available: %r' % sets_available) # logger.info('Sets: %r' % which_sets) which_sets_int = expand_string(which_sets, options=sets_available) if not which_sets_int: msg = 'Specified sets %r not found.' % which_sets msg += ' Available: %s' % sets_available raise UserError(msg) # logger.info('Expanded: %r' % which_sets) for x in which_sets_int: if not x in sets_available: msg = 'Set %r not available.' % x raise UserError(msg) if len(which_sets_int) == 1: combid = which_sets[0] else: combid = '-'.join(which_sets) # Create the new root root = data_central.root root_set = os.path.join(data_central.root, 'sets', combid) safe_makedirs(root_set) data_central_set = DataCentral(root_set) # add symbolic links to logs and config main_config = os.path.realpath(os.path.join(root, 'config')) set_config = os.path.join(root_set, 'config') safe_symlink(main_config, set_config) safe_makedirs(os.path.join(root_set, 'logs')) safe_symlink(os.path.join(root, 'logs'), os.path.join(root_set, 'logs', 'original')) storage = data_central_set.get_dir_structure().get_storage_dir() compmake_storage = os.path.join(storage, 'compmake') logger.debug('Using storage directory %r.' % friendly_path(compmake_storage)) use_filesystem(compmake_storage) for id_set in which_sets: if len(which_sets) > 1: comp_prefix(id_set) try: spec = batch_config.sets[x] batch_set(data_central_set, id_set, spec) except ConfToolsException: msg = ('Bad configuration for the set %r with spec\n %s' % (id_set, pformat(spec))) logger.error(msg) raise if command: return batch_command(command) else: compmake_console() return 0
from mycomputations import func1, func2, draw from compmake import comp, comp_prefix for param1 in [1, 2, 3]: for param2 in [10, 11, 12]: comp_prefix("p1=%s-p2=%s" % (param1, param2)) # use job_id to override default naming res1 = comp(func1, param1, job_id="preparing") res2 = comp(func2, res1, param2, job_id="computing") comp(draw, res2, job_id="drawing")
from mycomputations import func1, func2, draw from compmake import comp, comp_prefix for param1 in [1, 2, 3]: for param2 in [10, 11, 12]: # Add a prefix to the job ids # for easy reference comp_prefix('p1=%s-p2=%s' % (param1,param2)) res1 = comp(func1, param1) res2 = comp(func2, res1, param2) comp(draw, res2)