def execute(context, step):
f = in_base_dir(context, step["file"])
if "stdin" in step:
stdin = in_base_dir(context, step["stdin"])
command_seq = shell_command_seq(step, f)
proc = subprocess.run(command_seq, stdin=open(stdin, "r"), stdout=subprocess.PIPE)
else:
command_seq = shell_command_seq(step, f)
proc = subprocess.run(command_seq, stdout=subprocess.PIPE)
record_stdout(context, step, proc)
return context
def execute_async(context, step):
f = in_base_dir(context, step["file"])
command_seq = shell_command_seq(step, f)
if "stdout" in step:
stdout = in_base_dir(context, step["stdout"])
proc = subprocess.Popen(command_seq, stdout=open(stdout, "w"), preexec_fn=os.setsid)
else:
proc = subprocess.Popen(command_seq, preexec_fn=os.setsid)
proc_id = uuid.uuid4()
context["procs"][proc_id] = proc
return context
def make_file(context, step):
f = in_base_dir(context, step["file"])
with open(f, "r") as src:
with open(step["file"], "w") as dst:
for line in src:
dst.write(line)
return context
def record_stdout(context, step, proc):
stdout = str(proc.stdout, "UTF-8")
if stdout:
print(stdout)
if "stdout" in step:
with open(in_base_dir(context, step["stdout"]), "w") as f:
f.write(stdout)
def run_docker_proc(context, step):
input_sections = build_input_sections(context, step)
stdin_file = consolidate_input_files(input_sections)
f = in_base_dir(context, step["docker_bootup_file"])
with open(f, 'r') as handle:
base_cmd = shlex.split(handle.read())
cmd_seq = intercept_tty(base_cmd)
proc = subprocess.run(cmd_seq,
stdin=stdin_file,
stdout=subprocess.PIPE)
return ksql_proc_state(input_sections)
def write_spool_text(context, step, proc_state, temp_dir):
for group_name, spool_context in proc_state.items():
f = str(temp_dir + "/" + spool_context["spool_file_name"])
with open(f, "r") as handle:
content = shred_spool_text(handle.readlines())
stdout_dir = step["stdout"]["directory"]
full_dir = in_base_dir(context, "%s/%s" % (stdout_dir, group_name))
os.makedirs(full_dir)
for index, chunk in enumerate(content):
seq_file = "output-" + str(index) + ".log"
base_file = "%s/%s" % (full_dir, seq_file)
with open(base_file, "w") as output_file:
output_file.write("".join(chunk).lstrip())
def build_input_sections(context, step):
result = []
for block in step["stdin"]:
spool_file_name = str(uuid.uuid4()) + ".log"
spool_path = "/tmp/" + spool_file_name
f = in_base_dir(context, block["file"])
with open(f, "r") as handle:
commands = handle.readlines()
section = {
"group_name": get_file_name(f),
"spool_file_name": spool_file_name,
"spool_path": spool_path,
"spool_command": make_spool_command(spool_path),
"unspool_command": make_unspool_command(),
"commands": commands
}
result.append(section)
return result