def _retrieve_log_flat(queue, settings):
"""
Writes the given *log_filename* to *queue* in a flat format equivalent to::
./logviewer.py --flat log_filename
*settings* - A dict containing the *log_filename*, *colors_css*, and
*theme_css* to use when generating the HTML output.
"""
out_dict = {
'result': "",
'log': "",
'metadata': {},
}
# Local variables
out = []
spanstrip = re.compile(r'\s+\<\/span\>$')
user = settings['user']
users_dir = settings['users_dir']
log_filename = settings['log_filename']
logs_dir = os.path.join(users_dir, "logs")
log_path = os.path.join(logs_dir, log_filename)
if os.path.exists(log_path):
out_dict['metadata'] = get_or_update_metadata(log_path, user)
out_dict['metadata']['filename'] = log_filename
out_dict['result'] = "Success"
from io import BytesIO
# Use the terminal emulator to create nice HTML-formatted output
from terminal import Terminal
term = Terminal(rows=100, cols=300, em_dimensions=0)
io_obj = BytesIO()
flatten_log(log_path, io_obj)
io_obj.seek(0)
# Needed to emulate an actual term
flattened_log = io_obj.read().replace(b'\n', b'\r\n')
# NOTE: Using chunking below to emulate how a stream might actually be
# written to the terminal emulator. This is to prevent the emulator
# from thinking that any embedded files (like PDFs) are never going to
# end.
def chunker(s, n):
"""Produce `n`-character chunks from `s`."""
for start in range(0, len(s), n):
yield s[start:start+n]
for chunk in chunker(flattened_log, 499):
term.write(chunk)
scrollback, screen = term.dump_html()
# Join them together
log_lines = scrollback + screen
# rstrip the lines
log_lines = [a.rstrip() for a in log_lines]
# Fix things like "<span>whatever [lots of whitespace] </span>"
for i, line in enumerate(log_lines):
out.append(spanstrip.sub("</span>", line))
out_dict['log'] = out
term.clear_screen() # Ensure the function below works...
term.close_captured_fds() # Force clean up open file descriptors
else:
out_dict['result'] = _("ERROR: Log not found")
message = {'terminal:logging_log_flat': out_dict}
queue.put(message)
def _retrieve_log_flat(queue, settings):
"""
Writes the given *log_filename* to *queue* in a flat format equivalent to::
./logviewer.py --flat log_filename
*settings* - A dict containing the *log_filename*, *colors_css*, and
*theme_css* to use when generating the HTML output.
"""
out_dict = {
'result': "",
'log': "",
'metadata': {},
}
# Local variables
out = []
spanstrip = re.compile(r'\s+\<\/span\>$')
user = settings['user']
users_dir = settings['users_dir']
log_filename = settings['log_filename']
logs_dir = os.path.join(users_dir, "logs")
log_path = os.path.join(logs_dir, log_filename)
if os.path.exists(log_path):
out_dict['metadata'] = get_or_update_metadata(log_path, user)
out_dict['metadata']['filename'] = log_filename
out_dict['result'] = "Success"
from io import BytesIO
# Use the terminal emulator to create nice HTML-formatted output
from terminal import Terminal
term = Terminal(rows=100, cols=300, em_dimensions=0)
io_obj = BytesIO()
flatten_log(log_path, io_obj)
io_obj.seek(0)
# Needed to emulate an actual term
flattened_log = io_obj.read().replace(b'\n', b'\r\n')
# NOTE: Using chunking below to emulate how a stream might actually be
# written to the terminal emulator. This is to prevent the emulator
# from thinking that any embedded files (like PDFs) are never going to
# end.
def chunker(s, n):
"""Produce `n`-character chunks from `s`."""
for start in range(0, len(s), n):
yield s[start:start+n]
for chunk in chunker(flattened_log, 499):
term.write(chunk)
scrollback, screen = term.dump_html()
# Join them together
log_lines = scrollback + screen
# rstrip the lines
log_lines = [a.rstrip() for a in log_lines]
# Fix things like "<span>whatever [lots of whitespace] </span>"
for i, line in enumerate(log_lines):
out.append(spanstrip.sub("</span>", line))
out_dict['log'] = out
term.clear_screen() # Ensure the function below works...
term.close_captured_fds() # Force clean up open file descriptors
else:
out_dict['result'] = _("ERROR: Log not found")
message = {'terminal:logging_log_flat': out_dict}
queue.put(message)
def flatten_log(log_path, file_like, preserve_renditions=True, show_esc=False):
"""
Given a log file at *log_path*, write a string of log lines contained
within to *file_like*. Where *file_like* is expected to be any file-like
object with write() and flush() methods.
If *preserve_renditions* is True, CSI escape sequences for renditions will
be preserved as-is (e.g. font color, background, etc). This is to make the
output appear as close to how it was originally displayed as possible.
Besides that, it looks really nice =)
If *show_esc* is True, escape sequences and control characters will be
visible in the output. Trailing whitespace and escape sequences will not be
removed.
..note::
Converts our standard recording-based log format into something that
can be used with grep and similar search/filter tools.
"""
from terminal import Terminal, SPECIAL
metadata = get_log_metadata(log_path)
rows = metadata.get('rows', 24)
cols = metadata.get('columns', None)
if not cols:
# Try the old metadata format which used 'cols':
cols = metadata.get('cols', 80)
term = Terminal(rows=rows, cols=cols, em_dimensions=0)
out_line = u""
cr = False
# We skip the first frame, [1:] because it holds the recording metadata
for count, frame in enumerate(get_frames(log_path)):
if count == 0:
# Skip the first frame (it's just JSON-encoded metadata)
continue
# First 13 chars is the timestamp:
frame_time = float(frame.decode('UTF-8', 'ignore')[:13])
# Convert to datetime object
frame_time = datetime.fromtimestamp(frame_time/1000)
if show_esc:
frame_time = frame_time.strftime(u'\x1b[0m%b %m %H:%M:%S')
else: # Renditions preserved == I want pretty. Make the date bold:
frame_time = frame_time.strftime(u'\x1b[0;1m%b %m %H:%M:%S\x1b[m')
if not show_esc:
term.write(frame[14:])
if term.capture:
# Capturing a file... Keep feeding it frames until complete
continue
elif term.captured_files:
for line in term.screen:
# Find all the characters that come before/after the capture
for char in line:
if ord(char) >= SPECIAL:
adjusted = escape_escape_seq(out_line, rstrip=True)
adjusted = frame_time + u' %s\n' % adjusted
file_like.write(adjusted.encode('utf-8'))
out_line = u""
if char in term.captured_files:
captured_file = term.captured_files[char].file_obj
captured_file.seek(0)
file_like.write(captured_file.read())
file_like.write(b'\n')
del captured_file
term.clear_screen()
term.close_captured_fds() # Instant cleanup
else:
out_line += char
if not out_line:
continue
adjusted = frame_time + u' %s\n' % out_line.strip()
file_like.write(adjusted.encode('utf-8'))
out_line = u""
continue
else:
term.clear_screen()
frame = frame.decode('UTF-8', 'ignore')
for char in frame[14:]:
if '\x1b[H\x1b[2J' in out_line: # Clear screen sequence
# Handle the clear screen (usually ctrl-l) by outputting
# a new log entry line to avoid confusion regarding what
# happened at this time.
out_line += u"^L" # Clear screen is a ctrl-l or equivalent
if show_esc:
adjusted = raw(out_line)
else:
adjusted = escape_escape_seq(out_line, rstrip=True)
adjusted = frame_time + u' %s\n' % adjusted
file_like.write(adjusted.encode('utf-8'))
out_line = u""
continue
if char == u'\n':
if show_esc:
adjusted = raw(out_line)
else:
adjusted = escape_escape_seq(out_line, rstrip=True)
if not adjusted:
out_line = u"" # Skip empty lines
continue
adjusted = frame_time + u' %s\n' % adjusted
file_like.write(adjusted.encode('utf-8'))
out_line = u""
cr = False
elif char == u'\r':
# Carriage returns need special handling. Make a note of it
cr = True
else:
# \r without \n means that characters were (likely)
# overwritten. This usually happens when the user gets to
# the end of the line (which would create a newline in the
# terminal but not necessarily the log), erases their
# current line (e.g. ctrl-u), or an escape sequence modified
# the line in-place. To clearly indicate what happened we
# insert a '^M' and start a new line so as to avoid
# confusion over these events.
if cr:
out_line += "^M"
file_like.write((frame_time + u' ').encode('utf-8'))
if show_esc:
adjusted = raw(out_line)
else:
adjusted = escape_escape_seq(out_line, rstrip=True)
file_like.write((adjusted + u'\n').encode('utf-8'))
out_line = u""
out_line += char
cr = False
file_like.flush()
del term
def flatten_log(log_path, file_like, preserve_renditions=True, show_esc=False):
"""
Given a log file at *log_path*, write a string of log lines contained
within to *file_like*. Where *file_like* is expected to be any file-like
object with write() and flush() methods.
If *preserve_renditions* is True, CSI escape sequences for renditions will
be preserved as-is (e.g. font color, background, etc). This is to make the
output appear as close to how it was originally displayed as possible.
Besides that, it looks really nice =)
If *show_esc* is True, escape sequences and control characters will be
visible in the output. Trailing whitespace and escape sequences will not be
removed.
NOTE: Converts our standard recording-based log format into something that
can be used with grep and similar search/filter tools.
"""
import gzip
from terminal import Terminal, SPECIAL
term = Terminal(rows=100, cols=300, em_dimensions=0)
encoded_separator = SEPARATOR.encode('UTF-8')
out_line = u""
cr = False
# We skip the first frame, [1:] because it holds the recording metadata
for count, frame in enumerate(get_frames(log_path)):
if count == 0:
# Skip the first frame (it's just JSON-encoded metadata)
continue
# First 13 chars is the timestamp:
frame_time = float(frame.decode('UTF-8', 'ignore')[:13])
# Convert to datetime object
frame_time = datetime.fromtimestamp(frame_time / 1000)
if show_esc:
frame_time = frame_time.strftime(u'\x1b[0m%b %m %H:%M:%S')
else: # Renditions preserved == I want pretty. Make the date bold:
frame_time = frame_time.strftime(u'\x1b[0;1m%b %m %H:%M:%S\x1b[m')
if not show_esc:
term.write(frame[14:])
if term.capture:
# Capturing a file... Keep feeding it frames until complete
continue
elif term.captured_files:
for line in term.screen:
# Find all the characters that come before/after the capture
for char in line:
if ord(char) >= SPECIAL:
adjusted = escape_escape_seq(out_line, rstrip=True)
adjusted = frame_time + u' %s\n' % adjusted
file_like.write(adjusted.encode('utf-8'))
out_line = u""
if char in term.captured_files:
captured_file = term.captured_files[char].file_obj
captured_file.seek(0)
file_like.write(captured_file.read())
file_like.write(b'\n')
del captured_file
term.clear_screen()
term.close_captured_fds() # Instant cleanup
#term = Terminal(rows=100, cols=300, em_dimensions=0)
else:
out_line += char
adjusted = frame_time + u' %s\n' % out_line.strip()
file_like.write(adjusted.encode('utf-8'))
out_line = u""
continue
else:
term.clear_screen()
frame = frame.decode('UTF-8', 'ignore')
for char in frame[14:]:
if '\x1b[H\x1b[2J' in out_line: # Clear screen sequence
# Handle the clear screen (usually ctrl-l) by outputting
# a new log entry line to avoid confusion regarding what
# happened at this time.
out_line += u"^L" # Clear screen is a ctrl-l or equivalent
if show_esc:
adjusted = raw(out_line)
else:
adjusted = escape_escape_seq(out_line, rstrip=True)
adjusted = frame_time + u' %s\n' % adjusted
file_like.write(adjusted.encode('utf-8'))
out_line = u""
continue
if char == u'\n':
if show_esc:
adjusted = raw(out_line)
else:
adjusted = escape_escape_seq(out_line, rstrip=True)
adjusted = frame_time + u' %s\n' % adjusted
file_like.write(adjusted.encode('utf-8'))
out_line = u""
cr = False
elif char == u'\r':
# Carriage returns need special handling. Make a note of it
cr = True
else:
# \r without \n means that characters were (likely)
# overwritten. This usually happens when the user gets to
# the end of the line (which would create a newline in the
# terminal but not necessarily the log), erases their
# current line (e.g. ctrl-u), or an escape sequence modified
# the line in-place. To clearly indicate what happened we
# insert a '^M' and start a new line so as to avoid
# confusion over these events.
if cr:
out_line += "^M"
file_like.write((frame_time + u' ').encode('utf-8'))
if show_esc:
adjusted = raw(out_line)
else:
adjusted = escape_escape_seq(out_line, rstrip=True)
file_like.write((adjusted + u'\n').encode('utf-8'))
out_line = u""
out_line += char
cr = False
file_like.flush()
del term
