def run_sweep_doodad( target, params, run_mode, mounts, test_one=False, docker_image='python:3', is_docker_interactive=False, return_output=False, verbose=False, postprocess_config_and_run_mode=lambda config, run_mode, idx: (config, run_mode), default_params=None): # build archive target_dir = os.path.dirname(target) target_mount_dir = os.path.join('target', os.path.basename(target_dir)) target_mount = mount.MountLocal(local_dir=target_dir, mount_point=target_mount_dir) mounts = list(mounts) + [target_mount] target_full_path = os.path.join(target_mount.mount_point, os.path.basename(target)) command = launch_api.make_python_command(target_full_path) print('Launching jobs with mode %s' % run_mode) results = [] njobs = 0 with archive_builder.temp_archive_file() as archive_file: archive = archive_builder.build_archive( archive_filename=archive_file, payload_script=command, verbose=verbose, docker_image=docker_image, is_docker_interactive=is_docker_interactive, use_nvidia_docker=run_mode.use_gpu, mounts=mounts) sweeper = Sweeper(params, default_params) for config in sweeper: config, run_mode = postprocess_config_and_run_mode( config, run_mode, njobs) if config is None: continue njobs += 1 cli_args = ' '.join( ['--%s %s' % (key, config[key]) for key in config]) cmd = archive + ' -- ' + cli_args result = run_mode.run_script(cmd, return_output=return_output, verbose=False) if return_output: result = archive_builder._strip_stdout(result) results.append(result) if test_one: break print('Launching completed for %d jobs' % njobs) run_mode.print_launch_message() return tuple(results)
def run_sweep_doodad_chunked(target, params, run_mode, mounts, num_chunks=10, docker_image='python:3', return_output=False, test_one=False, confirm=True, verbose=False): # build archive target_dir = os.path.dirname(target) target_mount_dir = os.path.join('target', os.path.basename(target_dir)) target_mount = mount.MountLocal(local_dir=target_dir, mount_point=target_mount_dir) mounts = list(mounts) + [target_mount] target_full_path = os.path.join(target_mount.mount_point, os.path.basename(target)) command = launch_api.make_python_command(target_full_path) print('Launching jobs with mode %s' % run_mode) results = [] njobs = 0 with archive_builder.temp_archive_file() as archive_file: archive = archive_builder.build_archive( archive_filename=archive_file, payload_script=command, verbose=verbose, docker_image=docker_image, use_nvidia_docker=run_mode.use_gpu, mounts=mounts) sweeper = Sweeper(params) chunks = chunker(sweeper, num_chunks, confirm=confirm) for chunk in chunks: command = '' for config in chunk: njobs += 1 cli_args = ' '.join( ['--%s %s' % (key, config[key]) for key in config]) single_command = archive + ' -- ' + cli_args command += '%s;' % single_command result = run_mode.run_script(command, return_output=return_output, verbose=False) if return_output: result = archive_builder._strip_stdout(result) results.append(result) if test_one: break print('Launching completed for %d jobs on %d machines' % (njobs, num_chunks)) run_mode.print_launch_message() return tuple(results)
def run_command( command, cli_args=None, mode=launch_mode.LocalMode(), mounts=tuple(), return_output=False, verbose=False, docker_image='ubuntu:18.04', singularity_image=None, container_type='docker', extra_container_flags='', ): """ Runs a shell command using doodad via a specified launch mode. Args: command (str): A shell command cli_args (str): Command line args to pass mode (LaunchMode): A LaunchMode object mounts (tuple): A list/tuple of Mount objects return_output (bool): If True, returns stdout as a string. Do not use if the output will be large. container_type (string): either 'docker' or 'singularity extra_container_flags (string): flags other than GPU flags to pass to docker run or singularity exec. Returns: A string output if return_output is True, else None """ with archive_builder.temp_archive_file() as archive_file: archive = archive_builder.build_archive(archive_filename=archive_file, payload_script=command, verbose=False, docker_image=docker_image, singularity_image=singularity_image, container_type=container_type, extra_container_flags=extra_container_flags, use_gpu_image=mode.use_gpu, mounts=mounts) cmd = archive if cli_args: cmd = archive + ' -- ' + cli_args result = mode.run_script(cmd, return_output=return_output, verbose=verbose) if return_output: result = archive_builder._strip_stdout(result) return result
def run_command(command, cli_args=None, mode=launch_mode.LocalMode(), mounts=tuple(), return_output=False, verbose=False, docker_image='ubuntu:18.04'): """ Runs a shell command using doodad via a specified launch mode. Args: command (str): A shell command cli_args (str): Command line args to pass mode (LaunchMode): A LaunchMode object mounts (tuple): A list/tuple of Mount objects return_output (bool): If True, returns stdout as a string. Do not use if the output will be large. Returns: A string output if return_output is True, else None """ with archive_builder.temp_archive_file() as archive_file: archive = archive_builder.build_archive(archive_filename=archive_file, payload_script=command, verbose=False, docker_image=docker_image, use_nvidia_docker=mode.use_gpu, mounts=mounts) cmd = archive if cli_args: cmd = archive + ' -- ' + cli_args result = mode.run_script(cmd, return_output=return_output, verbose=verbose) if return_output: result = archive_builder._strip_stdout(result) return result