def rlearn(config, parser): #@UnusedVariable
t0 = time.time()
""" Displays the learned DDS """
parser.add_option("-n", "--nthreads", help="Number of threads",
type='int', default='4')
parser.add_option("-r", "--nrefine", help="Number of time to refine learning",
type='int', default='2')
parser.add_option("-s", "--streams", help="Which streams to use.",
default="*")
parser.add_option("-i", "--comb", default="default")
parser.add_option("-l", "--learners", help="Learner config.", default="*")
parser.add_option("-o", "--output", help="Output directory",
default='out/dp-rlearn/')
parser.add_option("--sensels", default=None,
help="Sensel indices to use for debuging refining module")
parser.add_option("-c", "--command",
help="Command to pass to compmake for batch mode")
parser.add_option("--show", default=None, help="Name of learners to report")
options = parser.parse_options()
if options.show is not None:
diffeomorphism2d_continuous.make_report(options.show.split(','))
sys.exit()
nthreads = options.nthreads
nrefine = options.nrefine
learners = config.learners.expand_names(options.learners)
streams = config.streams.expand_names(options.streams)
if len(learners) > 1:
logger.warn('Multiple learners are not supported for now')
if len(streams) > 1:
logger.warn('Multiple streams are not supported for now')
id_comb = ",".join(streams) + "-" + ",".join(learners)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, 'reports'))
jobs_rlearn(config, rm, learners, streams, outdir, nthreads, nrefine, options.sensels)
# Time and report the learning
comp(learning_times_rlearn, outdir, learners, streams, nthreads, nrefine)
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
logger.info("Done after time: " + str(time.time() - t0) + ' seconds')
def main():
from compmake import comp, compmake_console, use_filesystem
use_filesystem(os.path.join(Const.signals_dir, 'compmake'))
for id_video, id_filter in itertools.product(Const.videos, Const.filters):
if should_process(id_video, id_filter):
comp(extract_signals, id_video, id_filter,
job_id='extract-%s-%s' % (id_video, id_filter))
compmake_console()
def dp_predstats_main(config, parser):
parser.add_option("-o", "--output", default='out/dp-pred-stats',
help="Output directory")
parser.add_option("-d", "--distances", default='*',
help="Comma-separated list of distances. Can use *.")
parser.add_option("-S", "--dds",
help="Comma-separated list of diffeosystems.")
parser.add_option("-s", "--streams",
help="Comma-separated list of streams.")
parser.add_option("-c", "--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
if not options.streams:
msg = 'Please specify streams using -s.'
raise UserError(msg)
if not options.dds:
msg = 'Please specify which discdds to use.'
raise UserError(msg)
distances = config.distances.expand_names(options.distances)
streams = config.streams.expand_names(options.streams)
dds = config.streams.expand_names(options.dds)
logger.info('Using distances: %s' % distances)
logger.info('Using streams: %s' % streams)
logger.info('Using discdds: %s' % dds)
outdir = '%s/%s' % (options.output, options.dds)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, "reports"))
# for id_discdds in dds:
create_predstats_jobs(config=config, distances=distances,
id_discdds=dds,
streams=streams, rm=rm, maxd=10)
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def main():
from compmake import comp, compmake_console, use_filesystem
use_filesystem(os.path.join(Const.signals_dir, 'compmake'))
for id_video, id_filter in itertools.product(Const.videos, Const.filters):
if should_process(id_video, id_filter):
comp(extract_signals,
id_video,
id_filter,
job_id='extract-%s-%s' % (id_video, id_filter))
compmake_console()
def dp_dist_stats(config, parser): #@UnusedVariable
""" Computes statistics for images distances for different plan steps. """
parser.add_option("-o",
"--output",
default='out/dp-dist-stats',
help="Output directory")
parser.add_option("-d",
"--distances",
default='*',
help="Comma-separated list of distances. Can use *.")
parser.add_option("-s",
"--streams",
default='*',
help="Comma-separated list of streams. Can use *.")
parser.add_option("-r",
"--repeat",
default=1,
type='int',
help="Repeat many times.")
parser.add_option("-c",
"--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
distances = natsorted(config.distances.expand_names(options.distances))
streams = natsorted(config.streams.expand_names(options.streams))
logger.info('Using distances: %s' % distances)
logger.info('Using streams: %s' % streams)
id_comb = ','.join(streams) + '-' + ','.join(distances)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, "reports"))
create_diststats_jobs(config=config,
distances=distances,
streams=streams,
rm=rm,
maxd=10)
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def main():
use_filesystem(os.path.join(Const.signals_dir, 'compmake_stats'))
# signals = list_signals() # only do the compound ones
# signals = Const.osets.keys()
for id_oset, id_filter, id_stat in itertools.product(
Const.osets, Const.filters, Const.stats):
signal = '%s-%s' % (id_oset, id_filter)
comp(compute_and_write_stats, signal, id_stat,
job_id='stats-%s-%s' % (signal, id_stat))
compmake_console()
def main():
use_filesystem(os.path.join(Const.signals_dir, 'compmake_join'))
sets = {}
for fname in Const.filters.keys():
mkname = lambda x: '%s-%s' % (x, fname)
for master, pieces in Const.osets.items():
sets[mkname(master)] = [mkname(x) for x in pieces]
for master, pieces in sets.items():
comp(join_signals, master, pieces, job_id='join-%s' % master)
compmake_console()
def dp_batch_main(config, parser):
""" Runs batch planning experiments from batch configuration files. """
parser.add_option("-o",
"--output",
default='out/dp-batch',
help="Output directory")
parser.add_option("-c",
"--command",
help="Command to pass to compmake for batch mode")
options, which = parser.parse()
if not which:
todo = config.sets.keys()
id_comb = 'all'
else:
todo = config.sets.expand_names(which)
id_comb = "+".join(sorted(todo))
logger.info('Batch sets to do: %s' % todo)
outdir = os.path.join(options.output, 'set-%s' % id_comb)
# Compmake storage for results
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
for id_set in todo:
logger.info('Instantiating batch set %s' % id_set)
spec = config.sets[id_set]
try:
algos = config.algos.expand_names(spec['algorithms'])
testcases = config.testcases.expand_names(spec['testcases'])
comp_prefix('%s' % id_set)
b_outdir = os.path.join(outdir, id_set)
create_bench_jobs(config=config,
algos=algos,
testcases=testcases,
outdir=b_outdir)
except:
logger.error('Error while instantiating batch\n%s' % pformat(spec))
raise
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def main():
use_filesystem(os.path.join(Const.signals_dir, 'compmake_stats'))
# signals = list_signals() # only do the compound ones
# signals = Const.osets.keys()
for id_oset, id_filter, id_stat in itertools.product(
Const.osets, Const.filters, Const.stats):
signal = '%s-%s' % (id_oset, id_filter)
comp(compute_and_write_stats,
signal,
id_stat,
job_id='stats-%s-%s' % (signal, id_stat))
compmake_console()
def main():
use_filesystem(os.path.join(Const.signals_dir, 'compmake_join'))
sets = {}
for fname in Const.filters.keys():
mkname = lambda x: '%s-%s' % (x, fname)
for master, pieces in Const.osets.items():
sets[mkname(master)] = [mkname(x) for x in pieces]
for master, pieces in sets.items():
comp(join_signals, master, pieces,
job_id='join-%s' % master)
compmake_console()
def plearn(config, parser):
""" Learn the diffeomorphisms in parallel. """
# parser.add_option("-i", "--id_image", help="ID image.", default='lena')
ncpus = multiprocessing.cpu_count()
parser.add_option("-n",
"--nthreads",
help="Number of threads",
type='int',
default=ncpus)
parser.add_option("-s",
"--streams",
help="Which streams to use.",
default="*")
parser.add_option("-i", "--comb", default="default")
parser.add_option("-l", "--learners", help="Learner config.", default="*")
parser.add_option("-o",
"--output",
help="Output directory",
default='out/dp-plearn/')
parser.add_option("-c",
"--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
nthreads = options.nthreads
learners = config.learners.expand_names(options.learners)
streams = config.streams.expand_names(options.streams)
id_comb = ",".join(streams) + "-" + ",".join(learners)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, 'reports'))
jobs_plearn(config, rm, learners, streams, outdir, nthreads)
rm.create_index_job()
# Time and report the learning
comp(learning_times_plearn, outdir, learners, streams, nthreads)
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def main():
outdir = "test/repman"
storage = os.path.join(outdir, "compmake")
use_filesystem(storage)
rm = ReportManager(outdir)
report = comp(make_rep1, "TestReport3")
report2 = comp(make_rep1, "TestReport4")
rm.add(report, "rep3")
rm.add(report2, "rep4")
rm.create_index_job()
read_rc_files()
compmake_console()
def dp_bench_main(config, parser): # @UnusedVariable
""" Runs a set of planning experiments. """
parser.add_option("-a",
"--algorithms",
default='*',
help="Comma-separated list of algorithms. Can use *.")
parser.add_option("-t",
"--testcases",
default='*',
help="Comma-separated list of algorithms. Can use *.")
parser.add_option("-o",
"--output",
default='out/dp-bench',
help="Output directory")
parser.add_option("-c",
"--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
algos = config.algos.expand_names(options.algorithms)
testcases = config.testcases.expand_names(options.testcases)
logger.info('Using %d algorithms: %s' % (len(algos), algos))
logger.info('Using %d testcases.' % (len(testcases)))
outdir = options.output
# Compmake storage for results
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
create_bench_jobs(config=config,
algos=algos,
testcases=testcases,
outdir=outdir)
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def main():
outdir = 'test/repman'
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
rm = ReportManager(outdir)
report = comp(make_rep1, 'TestReport3')
report2 = comp(make_rep1, 'TestReport4')
rm.add(report, 'rep3')
rm.add(report2, 'rep4')
rm.create_index_job()
read_rc_files()
compmake_console()
def dp_batch_main(config, parser):
""" Runs batch planning experiments from batch configuration files. """
parser.add_option("-o", "--output", default='out/dp-batch',
help="Output directory")
parser.add_option("-c", "--command",
help="Command to pass to compmake for batch mode")
options, which = parser.parse()
if not which:
todo = config.sets.keys()
id_comb = 'all'
else:
todo = config.sets.expand_names(which)
id_comb = "+".join(sorted(todo))
logger.info('Batch sets to do: %s' % todo)
outdir = os.path.join(options.output, 'set-%s' % id_comb)
# Compmake storage for results
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
for id_set in todo:
logger.info('Instantiating batch set %s' % id_set)
spec = config.sets[id_set]
try:
algos = config.algos.expand_names(spec['algorithms'])
testcases = config.testcases.expand_names(spec['testcases'])
comp_prefix('%s' % id_set)
b_outdir = os.path.join(outdir, id_set)
create_bench_jobs(config=config, algos=algos,
testcases=testcases, outdir=b_outdir)
except:
logger.error('Error while instantiating batch\n%s' % pformat(spec))
raise
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def main():
np.seterr(all='raise')
parser = OptionParser(usage=description)
parser.add_option("--db", help="Main data directory")
parser.add_option("--interactive", action="store_true", default=False,
help="Starts an interactive compmake session.")
parser.add_option("--outdir", help="Output directory")
(options, args) = parser.parse_args() #@UnusedVariable
try:
if args:
raise Exception('Spurious arguments %r.' % args)
if not options.db:
raise Exception('Please provide --db option')
if not options.outdir:
raise Exception('Please provide --outdir option')
except Exception as e:
logger.error('Error while parsing configuration.')
logger.error(str(e))
sys.exit(-1)
try:
compmake_dir = os.path.join(options.outdir, 'compmake')
use_filesystem(compmake_dir)
with safe_flydra_db_open(options.db) as db:
spontaneous_analysis(db, options.outdir)
compmake_console()
except Exception as e:
logger.error('Error while processing. Exception and traceback follow.')
logger.error(str(e))
logger.error(traceback.format_exc())
sys.exit(-2)
def uncert(config, parser):
parser.add_option("-S", "--dds", help="DDS sytem .")
parser.add_option("-c", "--command", \
help="Command to pass to compmake for batch mode")
parser.add_option("-o", "--output", help="Output directory", \
default='out/dp-precision/')
parser.add_option("-s", "--streams", help="Which streams to use.", \
default="*")
parser.add_option("-d", "--distances", default='L2,L2w', \
help="Distances id to use for comparing the diffeo systems")
parser.add_option("-l", "--length", default=9, type='int', \
help="Max length of test cases")
options = parser.parse_options()
dds = config.discdds.expand_names(options.dds)
streams = config.streams.expand_names(options.streams)
id_comb = ",".join(dds)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, 'reports'))
max_delta = options.length
distances = config.distances.expand_names(options.distances)
store = create_stats_jobs(config, dds, streams, distances, max_delta, outdir)
records = comp(make_records, store)
report = comp(report_stats, records, dds, streams, distances)
r0 = comp(empty_report)
rm.add(report, 'main', id_dds='dds')
rm.add(r0, 'empty')
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def uncert(config, parser):
parser.add_option("-S", "--dds", help="DDS sytem .")
parser.add_option("-c", "--command", \
help="Command to pass to compmake for batch mode")
parser.add_option("-o", "--output", help="Output directory", \
default='out/dp-precision/')
parser.add_option("-s", "--streams", help="Which streams to use.", \
default="*")
parser.add_option("-d", "--distances", default='L2,L2w', \
help="Distances id to use for comparing the diffeo systems")
parser.add_option("-l", "--length", default=9, type='int', \
help="Max length of test cases")
options = parser.parse_options()
dds = config.discdds.expand_names(options.dds)
streams = config.streams.expand_names(options.streams)
id_comb = ",".join(dds)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, 'reports'))
max_delta = options.length
distances = config.distances.expand_names(options.distances)
store = create_stats_jobs(config, dds, streams, distances, max_delta,
outdir)
records = comp(make_records, store)
report = comp(report_stats, records, dds, streams, distances)
r0 = comp(empty_report)
rm.add(report, 'main', id_dds='dds')
rm.add(r0, 'empty')
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def uncert(config, parser): # @UnusedVariable
parser.add_option("-s",
"--streams",
help="Which streams to use.",
default="*")
parser.add_option("-S", "--dds", help="DDS sytem .")
parser.add_option("-o",
"--output",
help="Output directory",
default='out/dp-uncert/')
parser.add_option("-c",
"--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
dds = config.discdds.expand_names(options.dds)
streams = config.streams.expand_names(options.streams)
id_comb = ",".join(dds) + "-" + ",".join(streams)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, 'reports'))
max_delta = 9
store = create_uncert_stats_jobs(config, dds, streams, max_delta, outdir)
records = comp(make_records, store)
report = comp(report_uncert_stats, records, dds)
r0 = comp(empty_report)
rm.add(report, 'main', id_dds='dds')
rm.add(r0, 'empty')
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def pysnip_make(dirname, compmake_command):
files = glob(os.path.join(dirname, '*.py'))
prefixes = [os.path.splitext(os.path.basename(f))[0] for f in files]
logger.info('Found %d snippets in directory %s' % (len(prefixes), dirname))
use_filesystem(os.path.join(dirname, '.compmake'))
ntodo = 0
for p in prefixes:
job = Job(dirname, p)
job_id = job.basename
current_state = None
if job_exists(job_id):
current_state = get_job_cache(job_id).state
if job.status == DONE_UPTODATE:
# logger.info('%s: done' % job.basename)
if current_state != Cache.DONE:
mark_as_done(job_id)
pass
elif job.status == FAILED:
# logger.info('%s: failed' % job.basename)
if current_state != Cache.FAILED:
mark_as_failed(job_id)
elif job.status == DONE_NEEDSUPDATE:
mark_as_notstarted(job_id)
# logger.info('%s: done (but needs update)' % job.basename)
pass
elif job.status == NOTSTARTED:
mark_as_notstarted(job_id)
# logger.info('%s: not started' % job.basename)
pass
comp(run_job, job, job_id=job_id)
if job.status != DONE_UPTODATE:
ntodo += 1
# logger.info('%d/%d jobs to do' % (ntodo, len(prefixes)))
batch_command('stats')
if compmake_command is not None:
return batch_command(compmake_command)
else:
compmake_console()
return 0
def dp_dist_stats(config, parser): #@UnusedVariable
""" Computes statistics for images distances for different plan steps. """
parser.add_option("-o", "--output", default='out/dp-dist-stats',
help="Output directory")
parser.add_option("-d", "--distances", default='*',
help="Comma-separated list of distances. Can use *.")
parser.add_option("-s", "--streams", default='*',
help="Comma-separated list of streams. Can use *.")
parser.add_option("-r", "--repeat", default=1, type='int',
help="Repeat many times.")
parser.add_option("-c", "--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
distances = natsorted(config.distances.expand_names(options.distances))
streams = natsorted(config.streams.expand_names(options.streams))
logger.info('Using distances: %s' % distances)
logger.info('Using streams: %s' % streams)
id_comb = ','.join(streams) + '-' + ','.join(distances)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, "reports"))
create_diststats_jobs(config=config, distances=distances, streams=streams,
rm=rm, maxd=10)
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def uncert(config, parser): # @UnusedVariable
parser.add_option("-s", "--streams", help="Which streams to use.",
default="*")
parser.add_option("-S", "--dds", help="DDS sytem .")
parser.add_option("-o", "--output", help="Output directory",
default='out/dp-uncert/')
parser.add_option("-c", "--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
dds = config.discdds.expand_names(options.dds)
streams = config.streams.expand_names(options.streams)
id_comb = ",".join(dds) + "-" + ",".join(streams)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, 'reports'))
max_delta = 9
store = create_uncert_stats_jobs(config, dds, streams, max_delta, outdir)
records = comp(make_records, store)
report = comp(report_uncert_stats, records, dds)
r0 = comp(empty_report)
rm.add(report, 'main', id_dds='dds')
rm.add(r0, 'empty')
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def plearn(config, parser):
""" Learn the diffeomorphisms in parallel. """
# parser.add_option("-i", "--id_image", help="ID image.", default='lena')
ncpus = multiprocessing.cpu_count()
parser.add_option("-n", "--nthreads", help="Number of threads",
type='int', default=ncpus)
parser.add_option("-s", "--streams", help="Which streams to use.",
default="*")
parser.add_option("-i", "--comb", default="default")
parser.add_option("-l", "--learners", help="Learner config.", default="*")
parser.add_option("-o", "--output", help="Output directory",
default='out/dp-plearn/')
parser.add_option("-c", "--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
nthreads = options.nthreads
learners = config.learners.expand_names(options.learners)
streams = config.streams.expand_names(options.streams)
id_comb = ",".join(streams) + "-" + ",".join(learners)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, 'reports'))
jobs_plearn(config, rm, learners, streams, outdir, nthreads)
rm.create_index_job()
# Time and report the learning
comp(learning_times_plearn, outdir, learners, streams, nthreads)
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def dp_bench_main(config, parser): # @UnusedVariable
""" Runs a set of planning experiments. """
parser.add_option("-a", "--algorithms", default='*',
help="Comma-separated list of algorithms. Can use *.")
parser.add_option("-t", "--testcases", default='*',
help="Comma-separated list of algorithms. Can use *.")
parser.add_option("-o", "--output", default='out/dp-bench',
help="Output directory")
parser.add_option("-c", "--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
algos = config.algos.expand_names(options.algorithms)
testcases = config.testcases.expand_names(options.testcases)
logger.info('Using %d algorithms: %s' % (len(algos), algos))
logger.info('Using %d testcases.' % (len(testcases)))
outdir = options.output
# Compmake storage for results
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
create_bench_jobs(config=config, algos=algos,
testcases=testcases, outdir=outdir)
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def idealize_uncert(config, parser):
'''
Assumes a constant displacement over the whole sensor domain
'''
parser.add_option("-S", "--dds", help="DDS sytem .")
parser.add_option("-c", "--command", \
help="Command to pass to compmake for batch mode")
parser.add_option("-o", "--output", help="Output directory", \
default='out/idealized-dds/')
options = parser.parse_options()
id_discdds = options.dds
dds = config.discdds.instance(id_discdds)
outdir = os.path.join(options.output, id_discdds)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
#
rm = ReportManager(os.path.join(outdir, 'reports'))
# Ideal
id_iu_discdds = 'iu-' + id_discdds
iu_dds = comp(_idealize_uncert, dds, job_id='idealize_uncert')
comp(save_results, id_iu_discdds, outdir, iu_dds)
diffeo_report = comp(report_dds, 'iu_dds-%s' % id_discdds, iu_dds)
rm.add(diffeo_report, 'iu-dds', id_learner='idealized-uncertainty')
# Relative
id_uur_discdds = 'uur-' + id_discdds
dds_copyr = copy.copy(dds)
uur_dds = comp(_update_uncert, dds_copyr, length_score_norm_relative,
job_id='update_uncert_relative')
comp(save_results, id_uur_discdds, outdir, uur_dds,
job_id='update_uncert_relative_save')
diffeo_report = comp(report_dds, 'uur-dds-%s' % id_discdds, uur_dds,
job_id='update_uncert_relative_report')
rm.add(diffeo_report, 'uur-dds', id_learner='updated-uncertainty-uur')
# Absolute
dds_copya = copy.copy(dds)
id_uua_discdds = 'uua-' + id_discdds
uua_dds = comp(_update_uncert, dds_copya, length_score_norm,
job_id='update_uncert_absolute')
comp(save_results, id_uua_discdds, outdir, uua_dds,
job_id='update_uncert_absolute_save')
diffeo_report = comp(report_dds, 'uua-dds-%s' % id_discdds, uua_dds,
job_id='update_uncert_absolute_report')
rm.add(diffeo_report, 'uua-dds', id_learner='updated-uncertainty-uua')
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
# pdb.set_trace()
compmake_console()
return 0
def pixlearn(config, parser):
parser.add_option("-n", "--nthreads", help="Number of threads",
type='int', default='4')
parser.add_option("--distribute", type='str', default='random',
help="Strategy to distribute sensles to threads")
parser.add_option("-s", "--id_stream", help="Which streams to use.",
default="*")
parser.add_option("-o", "--output", help="Output directory",
default='out/dp-pixlearn/')
parser.add_option("-l", "--id_learner", help="Learner config.", default="*")
parser.add_option("-c", "--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
nthreads = options.nthreads
id_learner = config.learners.expand_names(options.id_learner)
id_stream = config.streams.expand_names(options.id_stream)
id_comb = ",".join(id_stream) + "-" + ",".join(id_learner)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, 'reports'))
pdb.set_trace()
if options.distribute == 'random':
max_nsensels = 40 * 30
# Generate a list with range of all indexes and assign them to threads
all_indicies = np.array(range(max_nsensels))
dist = np.random.randint(nthreads, size=max_nsensels)
sensel_indexes = []
for i in range(nthreads):
sensel_indexes.append(all_indicies[dist == i])
if options.distribute == 'demo4':
sensel_indexes = [[0, 1, 2, 3,
40, 41, 42, 43,
80, 81, 82, 83,
120, 121, 122, 123],
[340, 341, 342, 343,
380, 381, 382, 383,
420, 421, 422, 423,
460, 461, 462, 463],
[650, 651, 652, 653,
690, 691, 692, 693,
730, 731, 732, 733,
770, 771, 772, 773],
[1076, 1077, 1078, 1079,
1116, 1117, 1118, 1119,
1156, 1157, 1158, 1159,
1196, 1197, 1198, 1199]]
if len(id_learner) > 1 or len(id_stream) > 1:
logger.warning('learners and streams after index 0 will be ignored')
id_learner = id_learner[0]
id_stream = id_stream[0]
commands = [[256, 0, 0], [-256, 0, 0]]
states = [[100]]
# # Parallel part of code
# Initiate parallel learning
group = []
for i in range(nthreads):
estimator_i = comp(sensel_group_learn, config, id_learner, id_stream,
sensel_indexes[i])
group.append(estimator_i)
diffeo_system_i = comp(estimator_summarize, estimator_i,
commands, states, 'diffeo' + str(i))
estimator_report = comp(report_estimator, 'estimator' + str(i), estimator_i)
rm.add(estimator_report, 'estimator' + str(i), id_learner=id_learner,
id_stream=id_stream)
diffeo_report = comp(report_dds, 'diffeo' + str(i), diffeo_system_i)
rm.add(diffeo_report, 'diffeo' + str(i), id_learner=id_learner,
id_stream=id_stream)
estimator_main = comp(join_estimators, group)
main_system = comp(estimator_summarize, estimator_main,
commands, states, 'diffeo' + str(i))
diffeo_report = comp(report_dds, 'dds-%s-%s' % (id_stream, id_learner),
main_system,
job_id='learn-%s-%s-summarize-report' % (id_stream, id_learner))
rm.add(comp(empty_report), 'empty')
rm.add(diffeo_report, 'dds', id_learner=id_learner, id_stream=id_stream)
rm.create_index_job()
if options.command:
# return batch_command(options.command)
batch_command(options.command)
else:
compmake_console()
def batch_process_manager(data_central, which_sets, command=None):
try:
import compmake # @UnusedImport
except:
logger.error('Compmake not installed; multiprocessor '
'processes not available.')
raise
from compmake import (comp_prefix, use_filesystem,
compmake_console, batch_command)
batch_config = BatchConfigMaster()
configs = data_central.get_dir_structure().get_config_directories()
for config in configs:
batch_config.load(config)
sets_available = batch_config.sets.keys()
# logger.info('Available: %r' % sets_available)
# logger.info('Sets: %r' % which_sets)
which_sets_int = expand_string(which_sets, options=sets_available)
if not which_sets_int:
msg = 'Specified sets %r not found.' % which_sets
msg += ' Available: %s' % sets_available
raise UserError(msg)
# logger.info('Expanded: %r' % which_sets)
for x in which_sets_int:
if not x in sets_available:
msg = 'Set %r not available.' % x
raise UserError(msg)
if len(which_sets_int) == 1:
combid = which_sets[0]
else:
combid = '-'.join(which_sets)
# Create the new root
root = data_central.root
root_set = os.path.join(data_central.root, 'sets', combid)
safe_makedirs(root_set)
data_central_set = DataCentral(root_set)
# add symbolic links to logs and config
main_config = os.path.realpath(os.path.join(root, 'config'))
set_config = os.path.join(root_set, 'config')
safe_symlink(main_config, set_config)
safe_makedirs(os.path.join(root_set, 'logs'))
safe_symlink(os.path.join(root, 'logs'),
os.path.join(root_set, 'logs', 'original'))
storage = data_central_set.get_dir_structure().get_storage_dir()
compmake_storage = os.path.join(storage, 'compmake')
logger.debug('Using storage directory %r.' % friendly_path(compmake_storage))
use_filesystem(compmake_storage)
for id_set in which_sets:
if len(which_sets) > 1:
comp_prefix(id_set)
try:
spec = batch_config.sets[x]
batch_set(data_central_set, id_set, spec)
except ConfToolsException:
msg = ('Bad configuration for the set %r with spec\n %s' %
(id_set, pformat(spec)))
logger.error(msg)
raise
if command:
return batch_command(command)
else:
compmake_console()
return 0
def report_main(args):
# np.seterr(all='raise')
parser = LenientOptionParser(usage=description)
parser.add_option("--db", help="Main data directory")
parser.add_option("--outdir", help="Output directory for reports")
parser.add_option("--datadir", help="Output directory for compmake files")
parser.add_option("--version_rows",
help="Table version ('kf' or 'smooth')")
parser.add_option("--version_saccades",
help="Table version ('kf', 'smooth', 'angvel')")
parser.add_option("--group", help="Sample group", default='nopost')
parser.add_option("--ncells_distance", type='int', default=20,
help="Discretization for distance")
parser.add_option("--ncells_axis_angle", type='int', default=36,
help="Discretization for axis angle")
parser.add_option("--compmake_command", default=None,
help="Execute the CompMake command and exit.")
parser.add_option("--pdf", default=False, action='store_true',
help="Uses PDF for the reports (slower).")
(options, args) = parser.parse_args(args) # @UnusedVariable
check_no_spurious(args)
check_mandatory(options,
['db', 'outdir', 'datadir',
'version_rows', 'version_saccades'])
if options.pdf:
logger.info('Using PDF for plots.')
RepRepDefaults.default_image_format = MIME_PDF
PlotParams.init_matplotlib()
confid = '%s-%s-%s-D%d-A%d' % (options.group,
options.version_rows,
options.version_saccades,
options.ncells_distance,
options.ncells_axis_angle)
compmake_dir = os.path.join(options.datadir, confid)
use_filesystem(compmake_dir)
logger.info('Storing computation in %r.' % compmake_dir)
arena_radius = ParamsEstimation.arena_radius
warnings.warn('Using hardcoded arena radius %s.' % arena_radius)
cells = DACells(
ncells_distance=options.ncells_distance,
ncells_axis_angle=options.ncells_axis_angle,
arena_radius=arena_radius,
min_distance=ParamsEstimation.min_distance,
bin_enlarge_angle=ParamsEstimation.bin_enlarge_angle,
bin_enlarge_dist=ParamsEstimation.bin_enlarge_dist)
stats = comp(get_group_density_stats,
options.db, options.group, options.version_rows,
cells)
saccades = comp(get_saccades_for_group,
options.db, options.group, options.version_saccades)
saccades_stats = comp(compute_histogram_saccades, saccades, cells)
joint_stats = comp(compute_joint_statistics, stats, saccades_stats)
joint_stats = comp(compute_visual_stimulus, joint_stats)
report = comp(report_stats, confid, stats, saccades_stats)
rd = os.path.join(options.outdir, 'images')
html = os.path.join(options.outdir, "%s.html" % confid)
comp(write_report, report, html, rd,
job_id='report_stats-write')
report_m = comp(report_models_choice, confid, joint_stats)
html = os.path.join(options.outdir, "%s_models.html" % confid)
comp(write_report, report_m, html, rd,
job_id='report_models_choice-write')
report_s = comp(report_visual_stimulus, confid, joint_stats,
job_id='report_stimulus')
html = os.path.join(options.outdir, "%s_stimulus.html" % confid)
comp(write_report, report_s, html, rd,
job_id='report_stimulus-write')
report_i = comp(report_intuitive, confid, joint_stats,
job_id='report_intuitive')
html = os.path.join(options.outdir, "%s_intuitive.html" % confid)
comp(write_report, report_i, html, rd,
job_id='report_intuitive-write')
comp(write_report, comp(report_saccades, confid, saccades,
job_id='report_saccades'),
html=os.path.join(options.outdir, "%s_saccades.html" % confid),
rd=rd, job_id='report_saccades-write')
comp(write_report, comp(report_traj, confid,
options.db, options.group, options.version_rows,
job_id='report_traj'),
html=os.path.join(options.outdir, "%s_traj.html" % confid),
rd=rd, job_id='report_traj-write')
if options.compmake_command is not None:
compmake.batch_command(options.compmake_command)
else:
compmake_console()
def dp_predstats_main(config, parser):
parser.add_option("-o",
"--output",
default='out/dp-pred-stats',
help="Output directory")
parser.add_option("-d",
"--distances",
default='*',
help="Comma-separated list of distances. Can use *.")
parser.add_option("-S",
"--dds",
help="Comma-separated list of diffeosystems.")
parser.add_option("-s",
"--streams",
help="Comma-separated list of streams.")
parser.add_option("-c",
"--command",
help="Command to pass to compmake for batch mode")
options = parser.parse_options()
if not options.streams:
msg = 'Please specify streams using -s.'
raise UserError(msg)
if not options.dds:
msg = 'Please specify which discdds to use.'
raise UserError(msg)
distances = config.distances.expand_names(options.distances)
streams = config.streams.expand_names(options.streams)
dds = config.streams.expand_names(options.dds)
logger.info('Using distances: %s' % distances)
logger.info('Using streams: %s' % streams)
logger.info('Using discdds: %s' % dds)
outdir = '%s/%s' % (options.output, options.dds)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, "reports"))
# for id_discdds in dds:
create_predstats_jobs(config=config,
distances=distances,
id_discdds=dds,
streams=streams,
rm=rm,
maxd=10)
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
def rlearn(config, parser): #@UnusedVariable
t0 = time.time()
""" Displays the learned DDS """
parser.add_option("-n",
"--nthreads",
help="Number of threads",
type='int',
default='4')
parser.add_option("-r",
"--nrefine",
help="Number of time to refine learning",
type='int',
default='2')
parser.add_option("-s",
"--streams",
help="Which streams to use.",
default="*")
parser.add_option("-i", "--comb", default="default")
parser.add_option("-l", "--learners", help="Learner config.", default="*")
parser.add_option("-o",
"--output",
help="Output directory",
default='out/dp-rlearn/')
parser.add_option(
"--sensels",
default=None,
help="Sensel indices to use for debuging refining module")
parser.add_option("-c",
"--command",
help="Command to pass to compmake for batch mode")
parser.add_option("--show",
default=None,
help="Name of learners to report")
options = parser.parse_options()
if options.show is not None:
diffeomorphism2d_continuous.make_report(options.show.split(','))
sys.exit()
nthreads = options.nthreads
nrefine = options.nrefine
learners = config.learners.expand_names(options.learners)
streams = config.streams.expand_names(options.streams)
if len(learners) > 1:
logger.warn('Multiple learners are not supported for now')
if len(streams) > 1:
logger.warn('Multiple streams are not supported for now')
id_comb = ",".join(streams) + "-" + ",".join(learners)
outdir = os.path.join(options.output, id_comb)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
rm = ReportManager(os.path.join(outdir, 'reports'))
jobs_rlearn(config, rm, learners, streams, outdir, nthreads, nrefine,
options.sensels)
# Time and report the learning
comp(learning_times_rlearn, outdir, learners, streams, nthreads, nrefine)
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
compmake_console()
return 0
logger.info("Done after time: " + str(time.time() - t0) + ' seconds')
def idealize_uncert(config, parser):
'''
Assumes a constant displacement over the whole sensor domain
'''
parser.add_option("-S", "--dds", help="DDS sytem .")
parser.add_option("-c", "--command", \
help="Command to pass to compmake for batch mode")
parser.add_option("-o", "--output", help="Output directory", \
default='out/idealized-dds/')
options = parser.parse_options()
id_discdds = options.dds
dds = config.discdds.instance(id_discdds)
outdir = os.path.join(options.output, id_discdds)
storage = os.path.join(outdir, 'compmake')
use_filesystem(storage)
read_rc_files()
#
rm = ReportManager(os.path.join(outdir, 'reports'))
# Ideal
id_iu_discdds = 'iu-' + id_discdds
iu_dds = comp(_idealize_uncert, dds, job_id='idealize_uncert')
comp(save_results, id_iu_discdds, outdir, iu_dds)
diffeo_report = comp(report_dds, 'iu_dds-%s' % id_discdds, iu_dds)
rm.add(diffeo_report, 'iu-dds', id_learner='idealized-uncertainty')
# Relative
id_uur_discdds = 'uur-' + id_discdds
dds_copyr = copy.copy(dds)
uur_dds = comp(_update_uncert,
dds_copyr,
length_score_norm_relative,
job_id='update_uncert_relative')
comp(save_results,
id_uur_discdds,
outdir,
uur_dds,
job_id='update_uncert_relative_save')
diffeo_report = comp(report_dds,
'uur-dds-%s' % id_discdds,
uur_dds,
job_id='update_uncert_relative_report')
rm.add(diffeo_report, 'uur-dds', id_learner='updated-uncertainty-uur')
# Absolute
dds_copya = copy.copy(dds)
id_uua_discdds = 'uua-' + id_discdds
uua_dds = comp(_update_uncert,
dds_copya,
length_score_norm,
job_id='update_uncert_absolute')
comp(save_results,
id_uua_discdds,
outdir,
uua_dds,
job_id='update_uncert_absolute_save')
diffeo_report = comp(report_dds,
'uua-dds-%s' % id_discdds,
uua_dds,
job_id='update_uncert_absolute_report')
rm.add(diffeo_report, 'uua-dds', id_learner='updated-uncertainty-uua')
rm.create_index_job()
if options.command:
return batch_command(options.command)
else:
# pdb.set_trace()
compmake_console()
return 0
def main():
parser = OptionParser()
group = OptionGroup(parser, "Files and directories")
group.add_option("--outdir",
help='Directory with variables.pickle and where '
'the output will be placed.')
parser.add_option_group(group)
group = OptionGroup(parser, "Experiments options")
group.add_option("--fast",
default=False,
action='store_true',
help='Disables sanity checks.')
group.add_option("--set",
default='*',
help='[= %default] Which combinations to run.')
group.add_option("--seed",
default=None,
type='int',
help='[= %default] Seed for random number generator.')
parser.add_option_group(group)
group = OptionGroup(parser, "Compmake options")
group.add_option("--remake",
default=False,
action='store_true',
help='Remakes all (non interactive).')
group.add_option("--report",
default=False,
action='store_true',
help='Cleans and redoes all reports (non interactive).')
group.add_option(
"--report_stats",
default=False,
action='store_true',
help='Cleans and redoes the reports for the stats. (non interactive)')
parser.add_option_group(group)
(options, args) = parser.parse_args() #@UnusedVariable
np.random.seed(options.seed)
if options.fast:
disable_all()
assert not args
assert options.outdir is not None
available_algorithms, available_test_cases, available_sets = get_everything(
)
which = expand_string(options.set, list(available_sets.keys()))
if len(which) == 1:
compmake_storage = os.path.join(options.outdir, 'compmake', which[0])
else:
compmake_storage = os.path.join(options.outdir, 'compmake',
'common_storage')
use_filesystem(compmake_storage)
print('Staging creation of test cases reports')
test_cases = {}
test_case_reports = {}
def stage_test_case_report(tcid):
if not tcid in available_test_cases:
msg = ('Could not find test case %r \n %s' %
(tcid, available_test_cases.keys()))
raise Exception(msg)
if not tcid in test_cases:
command, args = available_test_cases[tcid]
job_id = 'test_case_data-%s' % tcid
test_cases[tcid] = comp(command, job_id=job_id, **args)
if not tcid in test_case_reports:
job_id = 'test_case-%s-report' % tcid
report = comp(create_report_tc,
tcid,
test_cases[tcid],
job_id=job_id)
job_id += '-write'
filename = os.path.join(options.outdir, 'test_cases',
'%s.html' % tcid)
comp(write_report, report, filename, job_id=job_id)
test_case_reports[tcid] = report
return test_case_reports[tcid]
# set of tuple (algo, test_case)
executions = {}
def stage_execution(tcid, algid):
stage_test_case_report(tcid)
key = (tcid, algid)
if not key in executions:
test_case = test_cases[tcid]
algo_class, algo_params = available_algorithms[algid]
job_id = 'solve-%s-%s-run' % (tcid, algid)
results = comp(run_combination,
tcid,
test_case,
algo_class,
algo_params,
job_id=job_id)
executions[key] = results
exc_id = '%s-%s' % (tcid, algid)
# Create iterations report
job_id = 'solve-%s-report' % exc_id
report = comp(create_report_execution,
exc_id,
tcid,
test_case,
algo_class,
algo_params,
results,
job_id=job_id)
job_id += '-write'
filename = os.path.join(options.outdir, 'executions',
'%s-%s.html' % (tcid, algid))
comp(write_report, report, filename, job_id=job_id)
return executions[key]
for comb_id in which:
comb = available_sets[comb_id]
alg_ids = expand_string(comb.algorithms, available_algorithms.keys())
tc_ids = expand_string(comb.test_cases, available_test_cases.keys())
print(
'Set %r has %d test cases and %d algorithms (~%d jobs in total).' %
(comb_id, len(alg_ids), len(tc_ids),
len(alg_ids) * len(tc_ids) * 2))
deps = {}
for t, a in itertools.product(tc_ids, alg_ids):
deps[(t, a)] = stage_execution(t, a)
job_id = 'tex-%s' % comb_id
comp(create_tables_for_paper,
comb_id,
tc_ids,
alg_ids,
deps,
job_id=job_id)
job_id = 'set-%s-report' % comb_id
report = comp(create_report_comb_stats,
comb_id,
tc_ids,
alg_ids,
deps,
job_id=job_id)
job_id += '-write'
filename = os.path.join(options.outdir, 'stats', '%s.html' % comb_id)
comp(write_report, report, filename, job_id=job_id)
if options.report or options.report_stats:
if options.report:
batch_command('clean *-report*')
elif options.report_stats:
batch_command('clean set-* tex*')
batch_command('parmake')
elif options.remake:
batch_command('clean *')
batch_command('make set-* tex-*')
else:
compmake_console()
def main():
parser = OptionParser()
group = OptionGroup(parser, "Files and directories")
group.add_option("--outdir",
help='Directory with variables.pickle and where '
'the output will be placed.')
parser.add_option_group(group)
group = OptionGroup(parser, "Experiments options")
group.add_option("--fast", default=False, action='store_true',
help='Disables sanity checks.')
group.add_option("--set", default='*',
help='[= %default] Which combinations to run.')
group.add_option("--seed", default=None, type='int',
help='[= %default] Seed for random number generator.')
parser.add_option_group(group)
group = OptionGroup(parser, "Compmake options")
group.add_option("--remake", default=False, action='store_true',
help='Remakes all (non interactive).')
group.add_option("--report", default=False, action='store_true',
help='Cleans and redoes all reports (non interactive).')
group.add_option("--report_stats", default=False, action='store_true',
help='Cleans and redoes the reports for the stats. (non interactive)')
parser.add_option_group(group)
(options, args) = parser.parse_args() #@UnusedVariable
np.random.seed(options.seed)
if options.fast:
disable_all()
assert not args
assert options.outdir is not None
available_algorithms, available_test_cases, available_sets = get_everything()
which = expand_string(options.set, list(available_sets.keys()))
if len(which) == 1:
compmake_storage = os.path.join(options.outdir, 'compmake', which[0])
else:
compmake_storage = os.path.join(options.outdir, 'compmake', 'common_storage')
use_filesystem(compmake_storage)
print('Staging creation of test cases reports')
test_cases = {}
test_case_reports = {}
def stage_test_case_report(tcid):
if not tcid in available_test_cases:
msg = ('Could not find test case %r \n %s' %
(tcid, available_test_cases.keys()))
raise Exception(msg)
if not tcid in test_cases:
command, args = available_test_cases[tcid]
job_id = 'test_case_data-%s' % tcid
test_cases[tcid] = comp(command, job_id=job_id, **args)
if not tcid in test_case_reports:
job_id = 'test_case-%s-report' % tcid
report = comp(create_report_tc,
tcid, test_cases[tcid], job_id=job_id)
job_id += '-write'
filename = os.path.join(options.outdir, 'test_cases', '%s.html' % tcid)
comp(write_report, report, filename, job_id=job_id)
test_case_reports[tcid] = report
return test_case_reports[tcid]
# set of tuple (algo, test_case)
executions = {}
def stage_execution(tcid, algid):
stage_test_case_report(tcid)
key = (tcid, algid)
if not key in executions:
test_case = test_cases[tcid]
algo_class, algo_params = available_algorithms[algid]
job_id = 'solve-%s-%s-run' % (tcid, algid)
results = comp(run_combination, tcid,
test_case, algo_class, algo_params,
job_id=job_id)
executions[key] = results
exc_id = '%s-%s' % (tcid, algid)
# Create iterations report
job_id = 'solve-%s-report' % exc_id
report = comp(create_report_execution, exc_id,
tcid,
test_case, algo_class, algo_params,
results, job_id=job_id)
job_id += '-write'
filename = os.path.join(options.outdir, 'executions',
'%s-%s.html' % (tcid, algid))
comp(write_report, report, filename, job_id=job_id)
return executions[key]
for comb_id in which:
comb = available_sets[comb_id]
alg_ids = expand_string(comb.algorithms, available_algorithms.keys())
tc_ids = expand_string(comb.test_cases, available_test_cases.keys())
print('Set %r has %d test cases and %d algorithms (~%d jobs in total).' %
(comb_id, len(alg_ids), len(tc_ids), len(alg_ids) * len(tc_ids) * 2))
deps = {}
for t, a in itertools.product(tc_ids, alg_ids):
deps[(t, a)] = stage_execution(t, a)
job_id = 'tex-%s' % comb_id
comp(create_tables_for_paper, comb_id, tc_ids, alg_ids, deps,
job_id=job_id)
job_id = 'set-%s-report' % comb_id
report = comp(create_report_comb_stats,
comb_id, tc_ids, alg_ids, deps, job_id=job_id)
job_id += '-write'
filename = os.path.join(options.outdir, 'stats', '%s.html' % comb_id)
comp(write_report, report, filename, job_id=job_id)
if options.report or options.report_stats:
if options.report:
batch_command('clean *-report*')
elif options.report_stats:
batch_command('clean set-* tex*')
batch_command('parmake')
elif options.remake:
batch_command('clean *')
batch_command('make set-* tex-*')
else:
compmake_console()
