def map_experiment_folder(experiment_folderpath, fn):
fo_paths = tb_fs.list_folders(
experiment_folderpath, recursive=False, use_relative_paths=False)
num_exps = len(
[p for p in fo_paths if tb_fs.path_last_element(p).startswith('cfg')])
ps = []
rs = []
for i in xrange(num_exps):
p = tb_fs.join_paths([experiment_folderpath, 'cfg%d' % i])
rs.append(fn(p))
ps.append(p)
return (ps, rs)
def create_runall_script(experiment_folderpath):
fo_names = tb_fs.list_folders(
experiment_folderpath, recursive=False, use_relative_paths=True)
num_exps = len(
[n for n in fo_names if tb_fs.path_last_element(n).startswith('cfg')])
# creating the script.
sc_lines = ['#!/bin/bash']
sc_lines += [
tb_fs.join_paths([experiment_folderpath,
"cfg%d" % i, 'run.sh']) for i in xrange(num_exps)
]
# creating the run all script.
out_filepath = tb_fs.join_paths([experiment_folderpath, 'run.sh'])
tb_io.write_textfile(out_filepath, sc_lines, with_newline=True)
st = os.stat(out_filepath)
exec_bits = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
os.chmod(out_filepath, st.st_mode | exec_bits)
def create_runall_script_with_parallelization(experiment_folderpath):
fo_names = tb_fs.list_folders(experiment_folderpath,
recursive=False,
use_relative_paths=True)
num_exps = len(
[n for n in fo_names if tb_fs.path_last_element(n).startswith('cfg')])
# creating the script.
sc_lines = [
'#!/bin/bash', 'if [ "$#" -lt 0 ] && [ "$#" -gt 3 ]; then',
' echo "Usage: run.sh [worker_id num_workers] [--force-rerun]"',
' exit 1', 'fi', 'force_rerun=0',
'if [ $# -eq 0 ] || [ $# -eq 1 ]; then', ' worker_id=0',
' num_workers=1', ' if [ $# -eq 1 ]; then',
' if [ "$1" != "--force-rerun" ]; then',
' echo "Usage: run.sh [worker_id num_workers] [--force-rerun]"',
' exit 1', ' else', ' force_rerun=1',
' fi', ' fi', 'else', ' worker_id=$1',
' num_workers=$2', ' if [ $# -eq 3 ]; then',
' if [ "$3" != "--force-rerun" ]; then',
' echo "Usage: run.sh [worker_id num_workers] [--force-rerun]"',
' exit 1', ' else', ' force_rerun=1',
' fi', ' fi', 'fi',
'if [ $num_workers -le $worker_id ] || [ $worker_id -lt 0 ]; then',
' echo "Invalid call: requires 0 <= worker_id < num_workers."',
' exit 1', 'fi'
'',
'num_exps=%d' % num_exps, 'i=0', 'while [ $i -lt $num_exps ]; do',
' if [ $(($i % $num_workers)) -eq $worker_id ]; then',
' if [ ! -f %s ] || [ $force_rerun -eq 1 ]; then' %
tb_fs.join_paths([experiment_folderpath, "cfg$i", 'results.json']),
' echo cfg$i',
' %s' %
tb_fs.join_paths([experiment_folderpath, "cfg$i", 'run.sh']),
' fi', ' fi', ' i=$(($i + 1))', 'done'
]
# creating the run all script.
out_filepath = tb_fs.join_paths([experiment_folderpath, 'run.sh'])
tb_io.write_textfile(out_filepath, sc_lines, with_newline=True)
st = os.stat(out_filepath)
exec_bits = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
os.chmod(out_filepath, st.st_mode | exec_bits)
def load_all_experiments(load_unfinished=False):
xs = [explore_experiment(p) for p in tb_fs.list_folders('experiments')]
return xs