def run():
ec2_mount = mount.MountS3(
s3_path='secret_output',
mount_point='/output'
)
local_mount = mount.MountLocal(
local_dir=TESTING_DIR,
mount_point='/data',
output=False
)
mounts = [local_mount, ec2_mount]
launcher = mode.EC2Autoconfig(
s3_bucket=S3_BUCKET,
s3_log_path='test_doodad/ec2_test',
instance_type='t3a.micro',
spot_price=0.03,
region='us-east-2',
ami_name='ami-0aa0375ca68b5554d',
)
#launch_api.run_command(
# command='cat /data/secret.txt > /output/secret.txt',
# mode=launcher,
# mounts=mounts,
# verbose=True
#)
launch_api.run_python(
target='testing/remote/py_print.py',
cli_args='--message=test_args123',
mode=launcher,
verbose=True
)
def __init__(
self,
mounts,
docker_img='python:3.5',
docker_output_dir='/data',
local_output_dir='data/docker',
python_cmd='python',
gcp_bucket_name=None,
gcp_image=None,
gcp_project=None,
):
self.image = docker_img
self.python_cmd = python_cmd
self.mode_local = doodad.mode.LocalDocker(image=docker_img)
# always include doodad
mounts.append(mount.MountLocal(local_dir=REPO_DIR, pythonpath=True))
self.docker_output_dir = docker_output_dir
self.mounts = mounts
self.mount_out_local = mount.MountLocal(local_dir=local_output_dir,
mount_point=docker_output_dir,
output=True)
self.mount_out_s3 = mount.MountS3(s3_path='exp_logs',
mount_point=docker_output_dir,
output=True)
self.gcp_bucket_name = gcp_bucket_name
self.gcp_image = gcp_image
self.gcp_project = gcp_project
self.mount_out_gcp = mount.MountGCP(gcp_path='exp_logs',
gcp_bucket_name=gcp_bucket_name,
mount_point=docker_output_dir,
output=True)
def run_sweep(run_experiment, sweep_params, exp_name, parser, instance_type='c4.xlarge'):
parser.add_argument('--mode', type=str, default='local',
help='Mode for running the experiments - local: runs on local machine, '
'ec2: runs on AWS ec2 cluster (requires a proper configuration file)')
parser.add_argument('--num_gpu', '-g', type=int, default=1,
help='Number of GPUs to use for running the experiments')
parser.add_argument('--exps_per_gpu', '-e', type=int, default=1,
help='Number of experiments per GPU simultaneously')
parser.add_argument('--num_cpu', '-c', type=int, default=multiprocessing.cpu_count(),
help='Number of threads to use for running experiments')
args = parser.parse_args(sys.argv[1:])
local_mount = mount.MountLocal(local_dir=config.BASE_DIR, pythonpath=True)
docker_mount_point = os.path.join(config.DOCKER_MOUNT_DIR, exp_name)
sweeper = launcher.DoodadSweeper([local_mount], docker_img=config.DOCKER_IMAGE,
docker_output_dir=docker_mount_point,
local_output_dir=os.path.join(config.DATA_DIR, 'local', exp_name))
if args.mode == 'ec2':
sweeper.mount_out_s3 = mount.MountS3(s3_path='', mount_point=docker_mount_point, output=True)
if args.mode == 'ec2':
print("\n" + "**********" * 10 + "\nexp_prefix: {}\nvariants: {}".format(exp_name, len(
list(itertools.product(*[value for value in sweep_params.values()])))))
if query_yes_no("Continue?"):
sweeper.run_sweep_ec2(run_experiment, sweep_params, bucket_name=config.S3_BUCKET_NAME,
instance_type=instance_type,
region='us-west-2', s3_log_name=exp_name, add_date_to_logname=False)
elif args.mode == 'local_docker':
mode_docker = dd.mode.LocalDocker(
image=sweeper.image,
)
run_sweep_doodad(run_experiment, sweep_params, run_mode=mode_docker,
mounts=sweeper.mounts)
elif args.mode == 'local':
sweeper.run_sweep_serial(run_experiment, sweep_params)
elif args.mode == 'local_par':
sweeper.run_sweep_parallel(run_experiment, sweep_params)
elif args.mode == 'multi_gpu':
run_sweep_multi_gpu(run_experiment, sweep_params, num_gpu=args.num_gpu, exps_per_gpu=args.exps_per_gpu)
elif args.mode == 'local_singularity':
mode_singularity = dd.mode.LocalSingularity(
image='~/maml_zoo.simg')
run_sweep_doodad(run_experiment, sweep_params, run_mode=mode_singularity,
mounts=sweeper.mounts)
else:
raise NotImplementedError
def launch(method,
params,
mode='ec2',
data_dependencies=dict(),
repeat=1,
instance_type='c4.xlarge'):
params['output_dir'] = [REMOTE_OUTPUT_DIR]
params['data_dir'] = [REMOTE_DATA_DIR]
if mode == 'local':
doodad_mode = pd.mode.Local()
params['output_dir'] = [LOCAL_OUTPUT_DIR]
elif mode == 'docker':
doodad_mode = pd.mode.LocalDocker(image='dibyaghosh/gcsl:0.1')
elif mode == 'ec2':
assert instance_type in instance_types
doodad_mode = pd.mode.EC2AutoconfigDocker(
image='dibyaghosh/gcsl:0.1',
region='us-west-1', # EC2 region
s3_log_prefix='gcsl', # Folder to store log files under
s3_log_name='gcsl',
terminate=True, # Whether to terminate on finishing job
**instance_types[instance_type])
data_mounts = [
mount.MountLocal(local_dir=osp.realpath(directory),
mount_point=osp.join(REMOTE_DATA_DIR, remote_name))
for remote_name, directory in data_dependencies.items()
]
if mode == 'local':
output_mounts = []
elif mode == 'docker' or mode == 'ssh':
output_dir = osp.join(LOCAL_OUTPUT_DIR, 'docker/')
output_mounts = [
mount.MountLocal(local_dir=output_dir,
mount_point=REMOTE_OUTPUT_DIR,
output=True)
]
elif mode == 'ec2':
output_mounts = [
mount.MountS3(s3_path='data',
mount_point=REMOTE_OUTPUT_DIR,
output=True)
]
mounts = code_mounts + data_mounts + output_mounts
hyper_sweep.run_sweep_doodad(method,
params,
doodad_mode,
mounts,
repeat=repeat)
def run(args, run_mode='local', storage_mode='local', dry_run=False):
local_mount = mount.MountLocal(
local_dir="/home/ashwin/Research/playground/softlearning",
mount_point='/code',
output=False)
local_mount2 = mount.MountLocal(local_dir="/home/ashwin/.mujoco",
mount_point='/root/.mujoco',
output=False)
# you can either run code on your machine or in the cloud
if run_mode == 'local':
launcher = mode.LocalMode()
elif run_mode == 'ec2':
launcher = mode.EC2Autoconfig(
s3_bucket="doodad-ashwin-redone2",
#s3_log_path='test_doodad/softlearning_test',
s3_log_path='softlearning_exps/',
instance_type='c4.large',
spot_price=0.03,
region='us-west-1',
ami_name='ami-874378e7',
)
results_mount_point = '/root/ray_results'
if storage_mode == 'local':
output_mount = mount.MountLocal(
local_dir="/home/ashwin/Research/results/",
mount_point=results_mount_point,
output=True)
elif storage_mode == 's3':
output_mount = mount.MountS3(s3_path='softlearning_output',
mount_point=results_mount_point,
output=True)
mounts = [local_mount, local_mount2, output_mount]
kwargs = {
#'command': "cat /code/secret.txt > /root/ray_results/secret.txt",
'command': command(args.domain, args.environment, args.exp_name),
'cli_args': "",
'mode': launcher,
'mounts': mounts,
'verbose': True,
'docker_image': 'abhiguptaberk/softlearningashwin:latest'
}
print(kwargs)
if not dry_run:
launch_api.run_command(**kwargs)
def test_s3(self):
""" Dry-run test for MountS3
This test doesn't actually run things it just makes sure nothing errors.
(we don't actually want to launch a test on EC2 and spend money)
"""
mnts = []
mnts.append(mount.MountS3(
s3_path='logs/mylogs',
dry=True,
))
payload_script = 'echo hello123'
archive = archive_builder_docker.build_archive(
payload_script=payload_script,
verbose=False,
docker_image='python:3',
mounts=mnts)
def create_mounts(
mode,
base_log_dir,
sync_interval=180,
local_input_dir_to_mount_point_dict=None,
):
if local_input_dir_to_mount_point_dict is None:
local_input_dir_to_mount_point_dict = {}
else:
raise NotImplementedError("TODO(vitchyr): Implement this")
mounts = [m for m in CODE_MOUNTS]
for dir, mount_point in local_input_dir_to_mount_point_dict.items():
mounts.append(
mount.MountLocal(
local_dir=dir,
mount_point=mount_point,
pythonpath=False,
))
if mode != 'local':
for m in NON_CODE_MOUNTS:
mounts.append(m)
if mode == 'ec2':
output_mount = mount.MountS3(
s3_path='',
mount_point=config.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
sync_interval=sync_interval,
)
elif mode == 'local':
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=None, # For purely local mode, skip mounting.
output=True,
)
elif mode == 'local_docker':
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=config.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
)
else:
raise NotImplementedError("Mode not supported: {}".format(mode))
mounts.append(output_mount)
return mounts
def run():
ec2_mount = mount.MountS3(s3_path='secret_output', mount_point='/output')
local_mount = mount.MountLocal(local_dir=TESTING_DIR,
mount_point='/data',
output=False)
mounts = [local_mount, ec2_mount]
launcher = mode.EC2Autoconfig(
s3_bucket=S3_BUCKET,
s3_log_path='test_doodad/gcp_test',
instance_type='c4.large',
spot_price=0.03,
region='us-west-1',
ami_name='ami-874378e7',
)
launch_api.run_command(command='cat /data/secret.txt > /output/secret.txt',
mode=launcher,
mounts=mounts,
verbose=True)
def __init__(
self,
mounts,
docker_img='python:3.5',
docker_output_dir='/data',
local_output_dir='data/docker',
):
self.image = docker_img
self.mode_local = doodad.mode.LocalDocker(image=docker_img)
# always include doodad
mounts.append(mount.MountLocal(local_dir=REPO_DIR, pythonpath=True))
self.mounts = mounts
self.mount_out_local = mount.MountLocal(local_dir=local_output_dir,
mount_point=docker_output_dir,
output=True)
self.mount_out_s3 = mount.MountS3(s3_path='exp_logs',
mount_point=docker_output_dir,
output=True)
def __init__(
self,
mounts=None,
docker_img='python:3',
docker_output_dir='/data',
local_output_dir='/data/docker',
s3_bucket_name=None,
gcp_bucket_name=None,
gcp_image=None,
gcp_project=None,
gcp_image_project=None,
):
if mounts is None:
mounts = []
self.image = docker_img
#self.python_cmd = python_cmd
self.mode_local = doodad.mode.LocalMode()
# always include doodad
#mounts.append(mount.MountLocal(local_dir=REPO_DIR, pythonpath=True))
self.docker_output_dir = docker_output_dir
self.mounts = mounts
self.mount_out_local = mount.MountLocal(local_dir=local_output_dir,
mount_point=docker_output_dir,
output=True)
self.gcp_bucket_name = gcp_bucket_name
self.gcp_image = gcp_image
self.gcp_project = gcp_project
self.gcp_image_project = gcp_image_project
if gcp_image_project is None:
self.gcp_image_project = gcp_project
self.mount_out_gcp = mount.MountGCP(gcp_path='exp_logs',
mount_point=docker_output_dir,
output=True)
self.s3_bucket = s3_bucket_name
self.mount_out_aws = mount.MountS3(s3_path='exp_logs',
mount_point=docker_output_dir,
output=True)
gpu=True,
terminate=False
)
MY_RUN_MODE = mode_ec2 # CHANGE THIS
# Set up code and output directories
OUTPUT_DIR = '/example/outputs' # this is the directory visible to the target
mounts = [
mount.MountLocal(local_dir=REPO_DIR, mount_point='/root/code/doodad', pythonpath=True), # Code
# mount.MountLocal(local_dir='/home/thanard/Downloads/rllab/sandbox/thanard/infoGAN', pythonpath=True),
mount.MountLocal(local_dir=os.path.join(EXAMPLES_DIR, 'secretlib'), pythonpath=True), # Code
]
if MY_RUN_MODE == mode_ec2:
output_mount = mount.MountS3(s3_path='outputs', mount_point=OUTPUT_DIR, output=True) # use this for ec2
else:
output_mount = mount.MountLocal(local_dir=os.path.join(EXAMPLES_DIR, 'tmp_output'),
mount_point=OUTPUT_DIR, output=True)
mounts.append(output_mount)
print(mounts)
THIS_FILE_DIR = os.path.realpath(os.path.dirname(__file__))
dd.launch_python(
target=os.path.join(THIS_FILE_DIR, 'app_main.py'),
# point to a target script. If running remotely, this will be copied over
mode=MY_RUN_MODE,
mount_points=mounts,
args={
'arg1': 50,
def create_mounts(
mode,
base_log_dir,
sync_interval=180,
local_input_dir_to_mount_point_dict=None,
):
if mode == 'sss':
code_mounts = SSS_CODE_MOUNTS
non_code_mounts = SSS_NON_CODE_MOUNTS
else:
code_mounts = CODE_MOUNTS
non_code_mounts = NON_CODE_MOUNTS
if local_input_dir_to_mount_point_dict is None:
local_input_dir_to_mount_point_dict = {}
else:
raise NotImplementedError("TODO(vitchyr): Implement this")
mounts = [m for m in code_mounts]
for dir, mount_point in local_input_dir_to_mount_point_dict.items():
mounts.append(mount.MountLocal(
local_dir=dir,
mount_point=mount_point,
pythonpath=False,
))
if mode != 'local':
for m in non_code_mounts:
mounts.append(m)
if mode == 'ec2':
output_mount = mount.MountS3(
s3_path='',
mount_point=conf.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
sync_interval=sync_interval,
include_types=('*.txt', '*.csv', '*.json', '*.gz', '*.tar',
'*.log', '*.pkl', '*.mp4', '*.png', '*.jpg',
'*.jpeg', '*.patch'),
)
elif mode == 'gcp':
output_mount = mount.MountGCP(
gcp_path='',
mount_point=conf.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
gcp_bucket_name=conf.GCP_BUCKET_NAME,
sync_interval=sync_interval,
include_types=('*.txt', '*.csv', '*.json', '*.gz', '*.tar',
'*.log', '*.pkl', '*.mp4', '*.png', '*.jpg',
'*.jpeg', '*.patch'),
)
elif mode in ['local', 'local_singularity', 'slurm_singularity', 'sss']:
# To save directly to local files (singularity does this), skip mounting
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=None,
output=True,
)
elif mode == 'local_docker':
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=conf.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
)
elif mode == 'ssh':
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=conf.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
)
else:
raise NotImplementedError("Mode not supported: {}".format(mode))
mounts.append(output_mount)
return mounts
type=int,
default=multiprocessing.cpu_count(),
help='Number of threads to use for running experiments')
args = parser.parse_args()
print(config.BASE_DIR)
local_mount = mount.MountLocal(local_dir=config.BASE_DIR, pythonpath=True)
docker_mount_point = os.path.join(config.DOCKER_MOUNT_DIR, EXP_NAME)
sweeper = launcher.DoodadSweeper([local_mount],
docker_img=config.DOCKER_IMAGE,
docker_output_dir=docker_mount_point,
local_output_dir=os.path.join(
config.DATA_DIR, 'local', EXP_NAME))
sweeper.mount_out_s3 = mount.MountS3(s3_path='',
mount_point=docker_mount_point,
output=True)
if args.mode == 'ec2':
if query_yes_no("Continue?"):
sweeper.run_sweep_ec2(run_experiment, {'alg': [0]},
bucket_name=config.S3_BUCKET_NAME,
instance_type='c4.xlarge',
region='us-west-1',
s3_log_name=EXP_NAME,
add_date_to_logname=True)
elif args.mode == 'local_docker':
mode_docker = dd.mode.LocalDocker(image=sweeper.image, )
run_sweep_doodad(run_experiment, {'alg': [0]},
run_mode=mode_docker,
mounts=sweeper.mounts)
def create_mounts(mode,
base_log_dir,
sync_interval=180,
local_input_dir_to_mount_point_dict=None):
if mode == "sss":
code_mounts = SSS_CODE_MOUNTS
non_code_mounts = SSS_NON_CODE_MOUNTS
else:
code_mounts = CODE_MOUNTS
non_code_mounts = NON_CODE_MOUNTS
if local_input_dir_to_mount_point_dict is None:
local_input_dir_to_mount_point_dict = {}
else:
raise NotImplementedError("TODO(vitchyr): Implement this")
mounts = [m for m in code_mounts]
for dir, mount_point in local_input_dir_to_mount_point_dict.items():
mounts.append(
mount.MountLocal(local_dir=dir,
mount_point=mount_point,
pythonpath=False))
if mode != "local":
for m in non_code_mounts:
mounts.append(m)
if mode == "ec2":
output_mount = mount.MountS3(
s3_path="",
mount_point=conf.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
sync_interval=sync_interval,
include_types=(
"*.txt",
"*.csv",
"*.json",
"*.gz",
"*.tar",
"*.log",
"*.pkl",
"*.mp4",
"*.png",
"*.jpg",
"*.jpeg",
"*.patch",
),
)
elif mode == "gcp":
output_mount = mount.MountGCP(
gcp_path="",
mount_point=conf.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
gcp_bucket_name=conf.GCP_BUCKET_NAME,
sync_interval=sync_interval,
include_types=(
"*.txt",
"*.csv",
"*.json",
"*.gz",
"*.tar",
"*.log",
"*.pkl",
"*.mp4",
"*.png",
"*.jpg",
"*.jpeg",
"*.patch",
),
)
elif mode in ["local", "local_singularity", "slurm_singularity", "sss"]:
# To save directly to local files (singularity does this), skip mounting
output_mount = mount.MountLocal(local_dir=base_log_dir,
mount_point=None,
output=True)
elif mode == "local_docker":
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=conf.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
)
elif mode == "ssh":
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=conf.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
)
else:
raise NotImplementedError("Mode not supported: {}".format(mode))
mounts.append(output_mount)
return mounts
# Set up code and output directories
OUTPUT_DIR = '/home/richard/newnrl/noreward-rl-private/src/tmp/' # this is the directory visible to the target
REPO_DIR = '/home/richard/newnrl/'
mounts = [
mount.MountLocal(local_dir=REPO_DIR,
pythonpath=True,
filter_dir=[
'tmp/', 'mapgen2_take2/', 'testmaps', 'outdir',
'curiosity', 'doomFiles', 'a3c-agent-results-test'
]), # Code
# mount.MountLocal(local_dir=os.path.join(EXAMPLES_DIR, 'secretlib'), pythonpath=True), # Code
]
if MY_RUN_MODE == mode_ec2:
output_mount = mount.MountS3(s3_path='outputs',
mount_point=OUTPUT_DIR,
output=True,
sync_interval=900) # use this for ec2
else:
output_mount = mount.MountLocal(local_dir=os.path.join(
EXAMPLES_DIR, 'tmp_output'),
mount_point=OUTPUT_DIR,
output=True)
mounts.append(output_mount)
# print(mounts)
THIS_FILE_DIR = os.path.realpath(os.path.dirname(__file__))
dd.launch_python(
# target=os.path.join(THIS_FILE_DIR, 'app_main.py'), # point to a target script. If running remotely, this will be copied over
target=os.path.join(REPO_DIR, 'noreward-rl-private/src/train.py'),
mode=MY_RUN_MODE,
def create_mounts(
mode,
base_log_dir,
sync_interval=180,
local_input_dir_to_mount_point_dict=None,
):
if mode in {"sss", "htp"}:
code_mounts = SSS_CODE_MOUNTS
non_code_mounts = SSS_NON_CODE_MOUNTS
else:
code_mounts = CODE_MOUNTS
non_code_mounts = NON_CODE_MOUNTS
if local_input_dir_to_mount_point_dict is None:
local_input_dir_to_mount_point_dict = {}
else:
raise NotImplementedError("TODO(vitchyr): Implement this")
mounts = [m for m in code_mounts]
for dir, mount_point in local_input_dir_to_mount_point_dict.items():
mounts.append(
mount.MountLocal(
local_dir=dir,
mount_point=mount_point,
pythonpath=False,
)
)
if mode != "local":
for m in non_code_mounts:
mounts.append(m)
if mode == "ec2":
output_mount = mount.MountS3(
s3_path="",
mount_point=config.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
sync_interval=sync_interval,
include_types=config.AWS_FILE_TYPES_TO_SAVE,
)
elif mode == "gcp":
output_mount = mount.MountGCP(
gcp_path="",
mount_point=config.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
gcp_bucket_name=config.GCP_BUCKET_NAME,
sync_interval=sync_interval,
include_types=config.GCP_FILE_TYPES_TO_SAVE,
)
elif mode in {"local", "local_singularity", "slurm_singularity", "sss", "htp"}:
# To save directly to local files, skip mounting
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=None,
output=True,
)
elif mode == "local_docker":
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=config.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
)
elif mode == "ssh":
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=config.OUTPUT_DIR_FOR_DOODAD_TARGET,
output=True,
)
else:
raise NotImplementedError("Mode not supported: {}".format(mode))
mounts.append(output_mount)
return mounts
# Set up code and output directories
OUTPUT_DIR = '/mount/outputs' # this is the directory visible to the target
input_mounts = [
# mount.MountLocal(local_dir=REPO_DIR, pythonpath=True),
mount.MountLocal(local_dir='~/install/rllab', pythonpath=True), # rllab
# mount.MountLocal(local_dir='~/install/gym/.mujoco', mount_point='/root/.mujoco'), # mujoco
# mount.MountLocal(local_dir='~/code/doodad', pythonpath=True), # rllab
]
output_mounts = [
mount.MountLocal(local_dir='~/data/vitchyr',
mount_point=OUTPUT_DIR,
read_only=False), # mujoco
mount.MountS3(
s3_path="test",
s3_bucket="2-12-2017.railrl.vitchyr.rail.bucket",
mount_point=OUTPUT_DIR,
output=True,
)
]
mounts = input_mounts + output_mounts
script_path = os.path.join(THIS_FILE_DIR, 'test_script.py')
print(script_path)
pd.launch_python(
target=script_path,
# script. If running remotely, this will be copied over
#target='/media/sg2/scripts/swimmer_data.py',
mode=mode_ec2,
# mode=mode_ssh,
# mode=mode_local,
mount_points=mounts,
def run_experiment(
method_call,
mode='local',
exp_prefix='default',
seed=None,
variant=None,
exp_id=0,
unique_id=None,
prepend_date_to_exp_prefix=True,
use_gpu=False,
snapshot_mode='last',
snapshot_gap=1,
n_parallel=0,
base_log_dir=None,
sync_interval=180,
local_input_dir_to_mount_point_dict=None, # TODO(vitchyr): test this
):
"""
Usage:
```
def foo(variant):
x = variant['x']
y = variant['y']
logger.log("sum", x+y)
variant = {
'x': 4,
'y': 3,
}
run_experiment(foo, variant, exp_prefix="my-experiment")
```
Results are saved to
`base_log_dir/<date>-my-experiment/<date>-my-experiment-<unique-id>`
By default, the base_log_dir is determined by
`config.LOCAL_LOG_DIR/`
:param method_call: a function that takes in a dictionary as argument
:param mode: 'local', 'local_docker', or 'ec2'
:param exp_prefix: name of experiment
:param seed: Seed for this specific trial.
:param variant: Dictionary
:param exp_id: One experiment = one variant setting + multiple seeds
:param unique_id: If not set, the unique id is generated.
:param prepend_date_to_exp_prefix: If False, do not prepend the date to
the experiment directory.
:param use_gpu:
:param snapshot_mode: See rllab.logger
:param snapshot_gap: See rllab.logger
:param n_parallel:
:param base_log_dir: Will over
:param sync_interval: How often to sync s3 data (in seconds).
:param local_input_dir_to_mount_point_dict: Dictionary for doodad.
:return:
"""
try:
import doodad
import doodad.mode
import doodad.mount as mount
from doodad.utils import REPO_DIR
except ImportError:
return run_experiment_old(
method_call,
exp_prefix=exp_prefix,
seed=seed,
variant=variant,
time_it=True,
mode=mode,
exp_id=exp_id,
unique_id=unique_id,
prepend_date_to_exp_prefix=prepend_date_to_exp_prefix,
use_gpu=use_gpu,
snapshot_mode=snapshot_mode,
snapshot_gap=snapshot_gap,
n_parallel=n_parallel,
base_log_dir=base_log_dir,
periodic_sync_interval=sync_interval,
)
global ec2_okayed
global gpu_ec2_okayed
if local_input_dir_to_mount_point_dict is None:
local_input_dir_to_mount_point_dict = {}
else:
raise NotImplementedError("TODO(vitchyr): Implement this")
# Modify some of the inputs
if seed is None:
seed = random.randint(0, 100000)
if variant is None:
variant = {}
for key, value in ppp.recursive_items(variant):
# This check isn't really necessary, but it's to prevent myself from
# forgetting to pass a variant through dot_map_dict_to_nested_dict.
if "." in key:
raise Exception(
"Variants should not have periods in keys. Did you mean to "
"convert {} into a nested dictionary?".format(key)
)
if unique_id is None:
unique_id = str(uuid.uuid4())
if prepend_date_to_exp_prefix:
exp_prefix = time.strftime("%m-%d") + "-" + exp_prefix
variant['seed'] = str(seed)
variant['exp_id'] = str(exp_id)
variant['unique_id'] = str(unique_id)
logger.log("Variant:")
logger.log(json.dumps(ppp.dict_to_safe_json(variant), indent=2))
mode_str_to_doodad_mode = {
'local': doodad.mode.Local(),
'local_docker': doodad.mode.LocalDocker(
image=config.DOODAD_DOCKER_IMAGE,
),
'ec2': doodad.mode.EC2AutoconfigDocker(
image=config.DOODAD_DOCKER_IMAGE,
region='us-east-2',
instance_type='c4.large',
spot_price=0.03,
s3_log_prefix=exp_prefix,
s3_log_name="{}-id{}-s{}".format(exp_prefix, exp_id, seed),
),
}
if base_log_dir is None:
base_log_dir = config.LOCAL_LOG_DIR
output_mount_point = config.OUTPUT_DIR_FOR_DOODAD_TARGET
mounts = [
mount.MountLocal(local_dir=REPO_DIR, pythonpath=True),
]
for code_dir in config.CODE_DIRS_TO_MOUNT:
mounts.append(mount.MountLocal(local_dir=code_dir, pythonpath=True))
for dir, mount_point in local_input_dir_to_mount_point_dict.items():
mounts.append(mount.MountLocal(
local_dir=dir,
mount_point=mount_point,
pythonpath=False,
))
if mode != 'local':
for non_code_mapping in config.DIR_AND_MOUNT_POINT_MAPPINGS:
mounts.append(mount.MountLocal(**non_code_mapping))
if mode == 'ec2':
if not ec2_okayed and not query_yes_no(
"EC2 costs money. Are you sure you want to run?"
):
sys.exit(1)
if not gpu_ec2_okayed and use_gpu:
if not query_yes_no(
"EC2 is more expensive with GPUs. Confirm?"
):
sys.exit(1)
gpu_ec2_okayed = True
ec2_okayed = True
output_mount = mount.MountS3(
s3_path='',
mount_point=output_mount_point,
output=True,
sync_interval=sync_interval,
)
# This will be over-written by the snapshot dir, but I'm setting it for
# good measure.
base_log_dir_for_script = output_mount_point
# The snapshot dir needs to be specified for S3 because S3 will
# automatically create the experiment director and sub-directory.
snapshot_dir_for_script = output_mount_point
elif mode == 'local':
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=None, # For purely local mode, skip mounting.
output=True,
)
base_log_dir_for_script = base_log_dir
# The snapshot dir will be automatically created
snapshot_dir_for_script = None
else:
output_mount = mount.MountLocal(
local_dir=base_log_dir,
mount_point=output_mount_point,
output=True,
)
base_log_dir_for_script = output_mount_point
# The snapshot dir will be automatically created
snapshot_dir_for_script = None
mounts.append(output_mount)
repo = git.Repo(os.getcwd())
code_diff = repo.git.diff(None)
if len(code_diff) > 5000:
logger.log("Git diff %d greater than 5000. Not saving diff." % len(code_diff))
code_diff = None
run_experiment_kwargs = dict(
exp_prefix=exp_prefix,
variant=variant,
exp_id=exp_id,
seed=seed,
use_gpu=use_gpu,
snapshot_mode=snapshot_mode,
snapshot_gap=snapshot_gap,
code_diff=code_diff,
commit_hash=repo.head.commit.hexsha,
script_name=main.__file__,
n_parallel=n_parallel,
base_log_dir=base_log_dir_for_script,
)
doodad.launch_python(
target=config.RUN_DOODAD_EXPERIMENT_SCRIPT_PATH,
mode=mode_str_to_doodad_mode[mode],
mount_points=mounts,
args={
'method_call': method_call,
'output_dir': snapshot_dir_for_script,
'run_experiment_kwargs': run_experiment_kwargs,
},
use_cloudpickle=True,
fake_display=True if mode != 'local' else False,
)
ec2_data_dir = '/home/ubuntu/rope_full'
DATA_DIR = args.data_dir # this is the directory visible to the target in docker
# Because imgs_skipped contains local data_dir paths.
PLANNING_DATA_DIR = args.planning_data_dir
mounts = [
mount.MountLocal(local_dir=REPO_DIR,
mount_point='/root/code/doodad',
pythonpath=True),
# mount.MountLocal(local_dir='/home/thanard/Downloads/rllab/sandbox/thanard/infoGAN', pythonpath=True),
mount.MountLocal(local_dir=project_dir,
filter_dir=excluded_dir,
pythonpath=True),
mount.MountEC2(ec2_path=ec2_data_dir, mount_point=DATA_DIR),
mount.MountS3(s3_path='',
mount_point=OUTPUT_DIR,
output=True,
include_types=('*.png', '*.log', '*.txt', '*.csv',
'*.json')),
]
local_img_tmp = os.path.join(args.data_dir, 'imgs_skipped_%d.pkl' % args.k)
if os.path.exists(local_img_tmp):
mounts.append(
mount.MountLocal(local_dir=local_img_tmp,
mount_point=os.path.join(
DATA_DIR, 'imgs_skipped_%d.pkl' % args.k)))
if kwargs['loadpath']:
for name in ['G', 'D', 'Gaussian_Posterior', 'Gaussian_Transition']:
for path, iter in zip(kwargs['loadpath'], kwargs['loaditer']):
mounts.append(
mount.MountLocal(
local_dir=os.path.join(path,