def test_capture_stdout(): buf = io.StringIO() ret_string, code = u.capture_stdout(["echo", "hello!"], file=buf) assert code == 0 # Verify that the stdout is reported to the supplied file, and that it's # captured by the function and returned correctly. assert ret_string == "hello!\n" assert buf.getvalue() == ret_string
def build_image(job_mode: c.JobMode, build_path: str, credentials_path: Optional[str] = None, adc_path: Optional[str] = None, no_cache: bool = False, dlvm: str = None, **kwargs) -> str: """Builds a Docker image by generating a Dockerfile and passing it to `docker build` via stdin. All output from the `docker build` process prints to stdout. Returns the image ID of the new docker container; if the command fails, throws on error with information about the command and any issues that caused the problem. """ with u.TempCopy(credentials_path, tmp_name=".caliban_default_creds.json") as creds: with u.TempCopy(adc_path, tmp_name=".caliban_adc_creds.json") as adc: cache_args = ["--no-cache"] if no_cache else [] cmd = ["docker", "build" ] + cache_args + ["--rm", "-f-", build_path] dockerfile = _dockerfile_template(job_mode, credentials_path=creds, adc_path=adc, dlvm=dlvm, **kwargs) joined_cmd = " ".join(cmd) logging.info("Running command: {}".format(joined_cmd)) try: output, ret_code = u.capture_stdout(cmd, input_str=dockerfile) if ret_code == 0: return docker_image_id(output) else: error_msg = "Docker failed with error code {}.".format( ret_code) raise DockerError(error_msg, cmd, ret_code) except subprocess.CalledProcessError as e: logging.error(e.output) logging.error(e.stderr)
def execute_jobs( job_specs: Iterable[JobSpec], dry_run: bool = False, ): '''executes a sequence of jobs based on job specs Arg: job_specs: specifications for jobs to be executed dry_run: if True, only print what would be done ''' with u.tqdm_logging() as orig_stream: pbar = tqdm.tqdm(logged_job_specs(job_specs), file=orig_stream, total=len(job_specs), ascii=True, unit="experiment", desc="Executing") for idx, job_spec in enumerate(pbar, 1): command = job_spec.spec['command'] logging.info(f'Running command: {" ".join(command)}') if not dry_run: _, ret_code = u.capture_stdout(command, "", u.TqdmFile(sys.stderr)) else: ret_code = 0 j = Job(spec=job_spec, container=job_spec.spec['container'], details={'ret_code': ret_code}, status=JobStatus.SUCCEEDED if ret_code == 0 else JobStatus.FAILED) local_callback(idx=idx, job=j) if dry_run: logging.info( t.yellow(f'\nTo build your image and execute these jobs, ' f'run your command again without {c.DRY_RUN_FLAG}\n')) return None
def test_capture_stdout_input(): ret_string, code = u.capture_stdout(["cat"], input_str="hello!") assert code == 0 assert ret_string.rstrip() == "hello!"