def test_capture_stdout():
buf = io.StringIO()
ret_string, code = u.capture_stdout(["echo", "hello!"], file=buf)
assert code == 0
# Verify that the stdout is reported to the supplied file, and that it's
# captured by the function and returned correctly.
assert ret_string == "hello!\n"
assert buf.getvalue() == ret_string
def build_image(job_mode: c.JobMode,
build_path: str,
credentials_path: Optional[str] = None,
adc_path: Optional[str] = None,
no_cache: bool = False,
dlvm: str = None,
**kwargs) -> str:
"""Builds a Docker image by generating a Dockerfile and passing it to `docker
build` via stdin. All output from the `docker build` process prints to
stdout.
Returns the image ID of the new docker container; if the command fails,
throws on error with information about the command and any issues that caused
the problem.
"""
with u.TempCopy(credentials_path,
tmp_name=".caliban_default_creds.json") as creds:
with u.TempCopy(adc_path, tmp_name=".caliban_adc_creds.json") as adc:
cache_args = ["--no-cache"] if no_cache else []
cmd = ["docker", "build"
] + cache_args + ["--rm", "-f-", build_path]
dockerfile = _dockerfile_template(job_mode,
credentials_path=creds,
adc_path=adc,
dlvm=dlvm,
**kwargs)
joined_cmd = " ".join(cmd)
logging.info("Running command: {}".format(joined_cmd))
try:
output, ret_code = u.capture_stdout(cmd, input_str=dockerfile)
if ret_code == 0:
return docker_image_id(output)
else:
error_msg = "Docker failed with error code {}.".format(
ret_code)
raise DockerError(error_msg, cmd, ret_code)
except subprocess.CalledProcessError as e:
logging.error(e.output)
logging.error(e.stderr)
def execute_jobs(
job_specs: Iterable[JobSpec],
dry_run: bool = False,
):
'''executes a sequence of jobs based on job specs
Arg:
job_specs: specifications for jobs to be executed
dry_run: if True, only print what would be done
'''
with u.tqdm_logging() as orig_stream:
pbar = tqdm.tqdm(logged_job_specs(job_specs),
file=orig_stream,
total=len(job_specs),
ascii=True,
unit="experiment",
desc="Executing")
for idx, job_spec in enumerate(pbar, 1):
command = job_spec.spec['command']
logging.info(f'Running command: {" ".join(command)}')
if not dry_run:
_, ret_code = u.capture_stdout(command, "",
u.TqdmFile(sys.stderr))
else:
ret_code = 0
j = Job(spec=job_spec,
container=job_spec.spec['container'],
details={'ret_code': ret_code},
status=JobStatus.SUCCEEDED
if ret_code == 0 else JobStatus.FAILED)
local_callback(idx=idx, job=j)
if dry_run:
logging.info(
t.yellow(f'\nTo build your image and execute these jobs, '
f'run your command again without {c.DRY_RUN_FLAG}\n'))
return None
def test_capture_stdout_input():
ret_string, code = u.capture_stdout(["cat"], input_str="hello!")
assert code == 0
assert ret_string.rstrip() == "hello!"
