def execute(context, step): f = in_base_dir(context, step["file"]) if "stdin" in step: stdin = in_base_dir(context, step["stdin"]) command_seq = shell_command_seq(step, f) proc = subprocess.run(command_seq, stdin=open(stdin, "r"), stdout=subprocess.PIPE) else: command_seq = shell_command_seq(step, f) proc = subprocess.run(command_seq, stdout=subprocess.PIPE) record_stdout(context, step, proc) return context
def execute_async(context, step): f = in_base_dir(context, step["file"]) command_seq = shell_command_seq(step, f) if "stdout" in step: stdout = in_base_dir(context, step["stdout"]) proc = subprocess.Popen(command_seq, stdout=open(stdout, "w"), preexec_fn=os.setsid) else: proc = subprocess.Popen(command_seq, preexec_fn=os.setsid) proc_id = uuid.uuid4() context["procs"][proc_id] = proc return context
def make_file(context, step): f = in_base_dir(context, step["file"]) with open(f, "r") as src: with open(step["file"], "w") as dst: for line in src: dst.write(line) return context
def record_stdout(context, step, proc): stdout = str(proc.stdout, "UTF-8") if stdout: print(stdout) if "stdout" in step: with open(in_base_dir(context, step["stdout"]), "w") as f: f.write(stdout)
def run_docker_proc(context, step): input_sections = build_input_sections(context, step) stdin_file = consolidate_input_files(input_sections) f = in_base_dir(context, step["docker_bootup_file"]) with open(f, 'r') as handle: base_cmd = shlex.split(handle.read()) cmd_seq = intercept_tty(base_cmd) proc = subprocess.run(cmd_seq, stdin=stdin_file, stdout=subprocess.PIPE) return ksql_proc_state(input_sections)
def write_spool_text(context, step, proc_state, temp_dir): for group_name, spool_context in proc_state.items(): f = str(temp_dir + "/" + spool_context["spool_file_name"]) with open(f, "r") as handle: content = shred_spool_text(handle.readlines()) stdout_dir = step["stdout"]["directory"] full_dir = in_base_dir(context, "%s/%s" % (stdout_dir, group_name)) os.makedirs(full_dir) for index, chunk in enumerate(content): seq_file = "output-" + str(index) + ".log" base_file = "%s/%s" % (full_dir, seq_file) with open(base_file, "w") as output_file: output_file.write("".join(chunk).lstrip())
def build_input_sections(context, step): result = [] for block in step["stdin"]: spool_file_name = str(uuid.uuid4()) + ".log" spool_path = "/tmp/" + spool_file_name f = in_base_dir(context, block["file"]) with open(f, "r") as handle: commands = handle.readlines() section = { "group_name": get_file_name(f), "spool_file_name": spool_file_name, "spool_path": spool_path, "spool_command": make_spool_command(spool_path), "unspool_command": make_unspool_command(), "commands": commands } result.append(section) return result