def _prettyformat_lines(lines): """ Pretty format lines of Eliot logs. """ for line in lines: data = json.loads(line) yield pretty_format(data) + '\n'
def read_logs(self): """Stream user-friendly structured logs of this execution. First attempt to stream using `eliot-tree` which provides a text tree-like structure of the execution given its logs. If `eliot-tree` fails (which happens when there is an eliot Task serialization bug in our code, a warning is yielded, followed by the output of `eliot-prettyprint`. Therefore, the output of this function is not guaranteed to be machine-readable. ..note:: This function may be called mid-run. """ from glob import glob import subprocess from .config import get_eliot_log_file filenames = glob(get_eliot_log_file(master_id=self.uuid).name + "*") eliottree_subp = subprocess.Popen(['eliot-tree', '--field-limit', '0'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) eliottree_failed = False with eliottree_subp.stdin: try: for filename in filenames: with open(filename, 'r') as file_: eliottree_subp.stdin.write(file_.read()) except (IOError, MemoryError): eliottree_failed = True with eliottree_subp.stdout: for line in eliottree_subp.stdout: yield line if eliottree_failed or (eliottree_subp.wait() != 0): # eliot-tree can fail on unfinished logging. We still want output # for debugging, so we use the less structured eliot-prettyprint from eliot.prettyprint import pretty_format from eliot._bytesjson import loads yield '\n`eliot-tree` failed to format output. ' \ 'Retrying with eliot-prettyprint:\n' for filename in filenames: yield "{}:\n".format(filename) with open(filename, 'r') as file_: for line in file_: yield pretty_format(loads(line))