def run():
launcher = doodad.AzureMode(azure_subscription_id=AZ_SUB_ID,
azure_storage_connection_str=AZ_CONN_STR,
azure_client_id=AZ_CLIENT_ID,
azure_authentication_key=AZ_SECRET,
azure_tenant_id=AZ_TENANT_ID,
azure_storage_container=AZ_CONTAINER,
log_path='doodad_test_experiment',
region='eastus',
instance_type='Standard_DS1_v2')
az_mount = doodad.MountAzure(
'azure_script_output',
mount_point='/output',
)
local_mount = doodad.MountLocal(local_dir=TESTING_DIR,
mount_point='/data',
output=False)
mounts = [local_mount, az_mount]
doodad.run_command(command='cat /data/secret.txt > /output/secret.txt',
mode=launcher,
mounts=mounts,
verbose=True)
def create_sweeper_and_output_mount(
mode,
log_path,
docker_image,
code_dirs_to_mount=config.CODE_DIRS_TO_MOUNT,
non_code_dirs_to_mount=config.NON_CODE_DIRS_TO_MOUNT,
remote_mount_configs=config.REMOTE_DIRS_TO_MOUNT,
use_gpu=False,
):
mounts = create_mounts(
code_dirs_to_mount=code_dirs_to_mount,
non_code_dirs_to_mount=non_code_dirs_to_mount,
remote_mount_configs=remote_mount_configs,
)
az_mount = doodad.MountAzure(
'',
mount_point='/output',
)
sweeper = DoodadSweeper(
mounts=mounts,
docker_img=docker_image,
azure_subscription_id=config.AZ_SUB_ID,
azure_storage_connection_str=config.AZ_CONN_STR,
azure_client_id=config.AZ_CLIENT_ID,
azure_authentication_key=config.AZ_SECRET,
azure_tenant_id=config.AZ_TENANT_ID,
azure_storage_container=config.AZ_CONTAINER,
mount_out_azure=az_mount,
local_output_dir=osp.join(
config.LOCAL_LOG_DIR,
log_path), # TODO: how to make this vary in local mode?
local_use_gpu=use_gpu,
)
# TODO: the sweeper should probably only have one output mount that is
# set rather than read based on the mode
if mode == 'azure':
output_mount = sweeper.mount_out_azure
elif mode == 'gcp':
output_mount = sweeper.mount_out_gcp
elif mode == 'here_no_doodad':
output_mount = sweeper.mount_out_local # this will be ignored
elif mode == 'local':
output_mount = sweeper.mount_out_local
else:
raise ValueError('Unknown mode: {}'.format(mode))
return sweeper, output_mount
def run():
log_path = 'sac-' + uuid.uuid4().hex[:4]
print(log_path)
launcher = doodad.AzureMode(
azure_subscription_id=SUBSC_ID,
azure_storage_container=CONTAINER_NAME,
azure_storage_connection_str=CONN_STR,
azure_client_id=CLIENT_ID,
azure_authentication_key=SECRET,
azure_tenant_id=TENANT_ID,
log_path=log_path,
region='eastus',
instance_type='Standard_DS1_v2',
)
mounts = [doodad.MountAzure('data', )]
doodad.run_command(command='echo foo >> /data/secret.txt',
mode=launcher,
mounts=mounts,
verbose=True)
def run():
launcher = doodad.AzureMode(azure_subscription_id=AZ_SUB_ID,
azure_storage_connection_str=AZ_CONN_STR,
azure_client_id=AZ_CLIENT_ID,
azure_authentication_key=AZ_SECRET,
azure_tenant_id=AZ_TENANT_ID,
azure_storage_container=AZ_CONTAINER,
log_path='doodad_gpu_test_experiment',
region='eastus',
use_gpu=True,
gpu_model='nvidia-tesla-k80',
num_gpu=1)
az_mount = doodad.MountAzure(
'azure_script_output',
mount_point='/output',
)
mounts = [az_mount]
doodad.run_command(command='nvidia-smi > /output/secret.txt',
mode=launcher,
mounts=mounts,
verbose=True)
import doodad
from doodad.wrappers.easy_launch import config
from doodad.wrappers.easy_launch import sweep_function
if __name__ == '__main__':
target = '/home/vitchyr/git/macaw/run.py'
az_mount = doodad.MountAzure(
'',
mount_point='/output',
)
mode = 'azure'
log_path = 'test_macaw'
docker_img = 'vitchyr/macaw-v0'
sweeper, output_mount = sweep_function._create_sweeper_and_output_mount(
mode, log_path, docker_img)
# import ipdb; ipdb.set_trace()
params = {
'env': ['ant_dir'],
}
sweeper.run_sweep_local(
target,
params,
return_output=True,
verbose=True,
)
# set_start_method('spawn')
# args = get_args()
#
# if args.instances == 1:
# if args.profile:
# import cProfile