def noop_env(config, environment, state, job, metadata):
with logutils.setup_logging_topic(config,
metadata,
state,
"step",
return_logger=True) as log:
log.info("state is: %s", state)
log.info("would be running this job: %s", job)
def test_process(config, environment, state, job, metadata):
with logutils.setup_logging_topic(config,
metadata,
state,
"step",
return_logger=True) as log:
log.info("a complicated test environment")
log.info("job: {}".format(job))
log.info("env: {}".format(environment))
log.info("state {}".format(state))
def run_packtivity(spec, parameters,state,metadata,pack_config, exec_config):
with logutils.setup_logging_topic(exec_config,metadata,state,'step',return_logger = True) as log:
parameters, state = finalize_inputs(parameters, state)
job, env = acquire_job_env(spec, parameters,state,metadata,pack_config)
if job and env:
try:
run_in_env(job, env,state,metadata,pack_config,exec_config)
except:
log.exception('job execution if job %s raise exception exception',metadata)
raise
pubdata = publish(spec['publisher'], parameters,state, pack_config)
pubdata = finalize_outputs(pubdata)
log.info('publishing data: %s',pubdata)
return pubdata
def docker_enc_handler(config, environment, state, job, metadata):
with logutils.setup_logging_topic(config,
metadata,
state,
"step",
return_logger=True) as log:
rspec = race_spec(config, state, environment, log, job)
log.debug("rspec is\n{}".format(json.dumps(rspec, indent=4)))
runtimes = {
"docker": run_containers_in_docker_runtime,
"singularity": run_containers_in_singularity_runtime,
}
run = runtimes[config.container_config.container_runtime()]
result = run(config, state, log, metadata, rspec)
return result
def localproc_env(config, environment, state, job, metadata):
with logutils.setup_logging_topic(config,
metadata,
state,
"step",
return_logger=True) as log:
olddir = os.path.realpath(os.curdir)
workdir = state.readwrite[0]
try:
log.info("changing to workdirectory %s", workdir)
utils.mkdir_p(workdir)
os.chdir(workdir)
shell = ["sh", "-c", str(job["command"])]
# shell = ['sh','-c','echo hello world']
log.info("running %s", shell)
subprocess.check_call(shell)
except:
log.exception("local job failed. job: %s", job)
raise RuntimeError("failed")
finally:
log.info("changing back to original directory %s", olddir)
os.chdir(olddir)
def run_packtivity(spec, parameters, state, metadata, pack_config,
exec_config):
with logutils.setup_logging_topic(exec_config,
metadata,
state,
"step",
return_logger=True) as log:
parameters, state = finalize_inputs(parameters, state)
job, env = acquire_job_env(spec, parameters, state, metadata,
pack_config)
if job and env:
try:
run_in_env(job, env, state, metadata, pack_config, exec_config)
except:
log.exception(
"job execution if job %s raise exception exception",
metadata)
raise
pubdata = publish(spec["publisher"], parameters, state, pack_config)
pubdata = finalize_outputs(pubdata)
log.info("publishing data: %s", pubdata)
return pubdata
def execute_and_tail_subprocess(
config,
metadata,
state,
log,
command_string,
stdin_content=None,
logging_topic="execution",
):
log.debug("command: \n%s", command_string)
log.debug("stdin if any: %s", stdin_content)
if config.dry_run():
return
try:
with logutils.setup_logging_topic(config,
metadata,
state,
logging_topic,
return_logger=True) as subproclog:
proc = None
if stdin_content:
log.debug("stdin: \n%s", stdin_content)
argv = shlex.split(command_string)
log.debug("argv: %s", argv)
proc = subprocess.Popen(
argv,
stdin=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
bufsize=1,
close_fds=True,
)
proc.stdin.write(stdin_content.encode("utf-8"))
proc.stdin.close()
else:
proc = subprocess.Popen(
shlex.split(command_string),
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
bufsize=1,
close_fds=True,
)
log.debug("started subprocess with pid %s. now wait to finish",
proc.pid)
time.sleep(0.5)
try: # some issues on some linux machines.. swallow exception
log.debug(
"process children: %s",
[
x for x in psutil.Process(proc.pid).children(
recursive=True)
],
)
except:
pass
for line in iter(proc.stdout.readline, b""):
subproclog.info(line.strip())
while proc.poll() is None:
pass
proc.stdout.close()
log.debug("container execution subprocess finished. return code: %s",
proc.returncode)
if proc.returncode:
log.error("non-zero return code raising exception")
raise subprocess.CalledProcessError(returncode=proc.returncode,
cmd=command_string)
log.debug(
"moving on from subprocess execution: {}".format(command_string))
except subprocess.CalledProcessError as exc:
log.exception("subprocess failed. code: %s, command %s",
exc.returncode, exc.cmd)
raise RuntimeError("failed container execution subprocess. %s",
command_string)
except:
log.exception("Unexpected error: %s", sys.exc_info())
raise
finally:
log.debug("finally for %s", command_string)