def StandardOutObserver(event):
if event["log_level"] not in levels:
return
if event.get("log_system", "-") == "-":
logSystem = "{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace", event.get("log_namespace", ''))
if format == "colour":
# Choose a colour depending on where the log came from.
if "Controller" in logSystem:
fore = Fore.BLUE
elif "Router" in logSystem:
fore = Fore.YELLOW
elif "Container" in logSystem:
fore = Fore.GREEN
else:
fore = Fore.WHITE
eventString = COLOUR_FORMAT.format(
fore, formatTime(event["log_time"]), logSystem, Fore.RESET,
formatEvent(event))
elif format == "nocolour":
eventString = NOCOLOUR_FORMAT.format(
formatTime(event["log_time"]), logSystem, formatEvent(event))
elif format == "syslogd":
eventString = SYSLOGD_FORMAT.format(logSystem, formatEvent(event))
print(eventString, file=_file)
def StandardErrorObserver(event):
if event["log_level"] not in levels:
return
if event.get("log_system", "-") == "-":
logSystem = "{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_source") is not None:
logSystem += " " + qual(event["log_source"].__class__)
if format == "colour":
# Errors are always red, no matter the system they came from.
eventString = COLOUR_FORMAT.format(
Fore.RED, formatTime(event["log_time"]), logSystem, Fore.RESET,
formatEvent(event))
elif format == "nocolour":
eventString = NOCOLOUR_FORMAT.format(
formatTime(event["log_time"]), logSystem, formatEvent(event))
elif format == "syslogd":
eventString = SYSLOGD_FORMAT.format(logSystem, formatEvent(event))
print(eventString, file=_stderr)
def StandardOutObserver(event):
if event["log_level"] not in levels:
return
if event["log_level"] == LogLevel.debug:
if event.get("txaio_trace", False) and not trace:
return
if event.get("log_system", "-") == "-":
logSystem = "{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace",
event.get("log_namespace", ''))
if event.get("log_category"):
format_string = _categories.get(event['log_category'])
if format_string:
event = event.copy()
event["log_format"] = format_string
if format == "standard":
FORMAT_STRING = STANDARD_FORMAT
elif format == "syslogd":
FORMAT_STRING = SYSLOGD_FORMAT
elif format == "none":
FORMAT_STRING = NONE_FORMAT
else:
assert False
if color:
# Choose a color depending on where the log came from.
if "Controller" in logSystem:
fore = Fore.BLUE
elif "Router" in logSystem:
fore = Fore.YELLOW
elif "Container" in logSystem:
fore = Fore.GREEN
else:
fore = Fore.WHITE
eventString = FORMAT_STRING.format(startcolor=fore,
time=formatTime(
event["log_time"]),
system=logSystem,
endcolor=Fore.RESET,
text=formatEvent(event))
else:
eventString = strip_ansi(
FORMAT_STRING.format(startcolor=u'',
time=formatTime(event["log_time"]),
system=logSystem,
endcolor=u'',
text=formatEvent(event)))
print(eventString, file=_file)
def StandardErrorObserver(event):
if event["log_level"] not in levels:
return
if event.get("log_system", u"-") == u"-":
logSystem = u"{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace",
event.get("log_namespace", ''))
if event.get("log_category"):
format_string = _categories.get(event['log_category'])
if format_string:
event = event.copy()
event["log_format"] = format_string
if event.get("log_format", None) is not None:
eventText = formatEvent(event)
else:
eventText = u""
if "log_failure" in event:
# This is a traceback. Print it.
eventText = eventText + event["log_failure"].getTraceback()
if format == "standard":
FORMAT_STRING = STANDARD_FORMAT
elif format == "syslogd":
FORMAT_STRING = SYSLOGD_FORMAT
elif format == "none":
FORMAT_STRING = NONE_FORMAT
else:
assert False
if colour:
# Errors are always red.
fore = Fore.RED
eventString = FORMAT_STRING.format(startcolour=fore,
time=formatTime(
event["log_time"]),
system=logSystem,
endcolour=Fore.RESET,
text=formatEvent(event))
else:
eventString = strip_ansi(
FORMAT_STRING.format(startcolour=u'',
time=formatTime(event["log_time"]),
system=logSystem,
endcolour=u'',
text=formatEvent(event)))
print(eventString, file=_file)
def StandardOutObserver(event):
if event["log_level"] not in levels:
return
if event["log_level"] == LogLevel.debug:
if event.get("txaio_trace", False) and not trace:
return
if event.get("log_system", "-") == "-":
logSystem = "{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace", event.get("log_namespace", ''))
if event.get("log_category"):
format_string = _categories.get(event['log_category'])
if format_string:
event = event.copy()
event["log_format"] = format_string
if format == "standard":
FORMAT_STRING = STANDARD_FORMAT
elif format == "syslogd":
FORMAT_STRING = SYSLOGD_FORMAT
elif format == "none":
FORMAT_STRING = NONE_FORMAT
else:
assert False
if colour:
# Choose a colour depending on where the log came from.
if "Controller" in logSystem:
fore = Fore.BLUE
elif "Router" in logSystem:
fore = Fore.YELLOW
elif "Container" in logSystem:
fore = Fore.GREEN
else:
fore = Fore.WHITE
eventString = FORMAT_STRING.format(
startcolour=fore, time=formatTime(event["log_time"]),
system=logSystem, endcolour=Fore.RESET,
text=formatEvent(event))
else:
eventString = strip_ansi(FORMAT_STRING.format(
startcolour=u'', time=formatTime(event["log_time"]),
system=logSystem, endcolour=u'',
text=formatEvent(event)))
print(eventString, file=_file)
def StandardErrorObserver(event):
if event["log_level"] not in levels:
return
if event.get("log_system", u"-") == u"-":
logSystem = u"{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace", event.get("log_namespace", ''))
if event.get("log_category"):
format_string = _categories.get(event['log_category'])
if format_string:
event = event.copy()
event["log_format"] = format_string
if event.get("log_format", None) is not None:
eventText = formatEvent(event)
else:
eventText = u""
if "log_failure" in event:
# This is a traceback. Print it.
eventText = eventText + event["log_failure"].getTraceback()
if format == "standard":
FORMAT_STRING = STANDARD_FORMAT
elif format == "syslogd":
FORMAT_STRING = SYSLOGD_FORMAT
elif format == "none":
FORMAT_STRING = NONE_FORMAT
else:
assert False
if colour:
# Errors are always red.
fore = Fore.RED
eventString = FORMAT_STRING.format(
startcolour=fore, time=formatTime(event["log_time"]),
system=logSystem, endcolour=Fore.RESET,
text=formatEvent(event))
else:
eventString = strip_ansi(FORMAT_STRING.format(
startcolour=u'', time=formatTime(event["log_time"]),
system=logSystem, endcolour=u'',
text=formatEvent(event)))
print(eventString, file=_file)
def wrapped(event):
try:
# If there is a failure, use the type and message to create the main msg.
if 'failure' in event:
f = event['failure']
parts = [f.type.__name__, f.getErrorMessage()]
msg = ' '.join(filter(None, parts))
# Otherwise use normal event formatting to create the main msg.
else:
msg = formatEvent(event)
new = OrderedDict([('level', level_name[event.pop('log_level')]),
('msg', msg), ('service', service),
('timestamp', _timestamp())])
if 'log_namespace' in event:
new['namespace'] = event.pop('log_namespace')
if 'log_system' in event:
new['system'] = event.pop('log_system')
# Keep all keys except the noise.
for k, v in sorted(event.items()):
if k not in noisey_keys:
new[k] = v
except Exception as e:
# Fallback to normal event processing
new = event
new['log_failure'] = str(e)
json(new)
def __call__(self, event):
# it seems if a twisted.logger.Logger() has .failure() called
# on it, the log_format will be None for the traceback after
# "Unhandled error in Deferred" -- perhaps this is a Twisted
# bug?
if event['log_format'] is None:
msg = u'{0} {1}{2}'.format(
formatTime(event["log_time"]),
failure_format_traceback(event['log_failure']),
os.linesep,
)
if self._encode:
msg = msg.encode('utf8')
self._file.write(msg)
else:
# although Logger will already have filtered out unwanted
# levels, bare Logger instances from Twisted code won't have.
if 'log_level' in event and self._acceptable_level(event['log_level']):
msg = u'{0} {1}{2}'.format(
formatTime(event["log_time"]),
formatEvent(event),
os.linesep,
)
if self._encode:
msg = msg.encode('utf8')
self._file.write(msg)
def __call__(self, event):
# Called by Twisted when delivering a log event to this observer.
# Format a message with four space-separated elements:
# - Fixed width, capitalized first letter of log level.
# - Fixed width, seconds and milliseconds of event timestamp.
# - Variable width, logger namespace.
# - Variable width, formatted message itself.
log_datetime = datetime.fromtimestamp(event['log_time'])
log_message = '%s %s %s %s' % (
event['log_level'].name[0].upper(),
log_datetime.strftime('%S.%f')[:6],
event.get('log_namespace', '-'),
logger.formatEvent(event),
)
# twisted.logger.formatEvent does not include eventual tracebacks but
# we want to show them: borrowed from t.l.formatEventAsClassicLogText
if 'log_failure' in event:
try:
traceback = event['log_failure'].getTraceback()
except Exception as e:
traceback = 'FAILED GETTING TRACEBACK: %r' % (e, )
traceback = textwrap.indent(traceback, 'C ###### ')
log_message = '\n'.join((log_message, traceback)).rstrip()
self._send_message_dict('log-message', {
'message': log_message,
})
def json_format(self, event):
error = bool(event.get("isError")) or "failure" in event
ts = event["log_time"]
if error:
severity = 3
else:
severity = 5
msg = {
"Hostname": HOSTNAME,
"Timestamp": ts * 1000 * 1000 * 1000,
"Type": "twisted:log",
"Severity": event.get("severity") or severity,
"EnvVersion": "2.0",
"Fields": {
k: v
for k, v in event.iteritems()
if k not in IGNORED_KEYS and type(v) in (str, unicode, list,
int, float)
},
"Logger": self.logger_name,
}
# Add the nicely formatted message
msg["Fields"]["message"] = formatEvent(event)
return json.dumps(msg, skipkeys=True) + "\n"
def _render(event):
if event.get("log_system", u"-") == u"-":
logSystem = u"{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace",
event.get("log_namespace", ''))
if event.get("log_format", None) is not None:
eventText = formatEvent(event)
else:
eventText = u""
if "log_failure" in event:
# This is a traceback. Print it.
eventText = eventText + event["log_failure"].getTraceback()
eventString = strip_ansi(
STANDARD_FORMAT.format(startcolor=u'',
time=formatTime(event["log_time"]),
system=logSystem,
endcolor=u'',
text=eventText)) + os.linesep
return eventString
def formatLogEvent(event, formatTime=formatTime):
eventText = formatEvent(event)
if not eventText:
return None
eventText = eventText.replace(u"\n", u"\n\t")
timeStamp = formatTime(event.get("log_time", None))
system = event.get("log_system", None)
level = event.get("log_level", None)
if level is None:
levelName = u"-"
else:
levelName = level.name
if system is None:
system = u"{namespace}".format(namespace=event.get(
"log_namespace", u"-"), )
else:
try:
system = str(system)
except Exception:
system = u"UNFORMATTABLE"
return u"{timeStamp} [{system}] {level}: {event}\n".format(
timeStamp=timeStamp,
system=system.ljust(35),
level=levelName.upper().ljust(8),
event=eventText,
)
def __call__(self, event):
"""
Make log entry and send it
@param event: An event.
@type event: L{dict}
"""
if not self.simplelogs_url:
return
message = formatEvent(event)
if "log_failure" in event:
try:
traceback = event["log_failure"].getTraceback()
except Exception:
traceback = u"(UNABLE TO OBTAIN TRACEBACK FROM EVENT)\n"
message = u"\n".join((message, traceback))
level = event['log_level'].name.lower()
if level == 'warn':
level = 'warning'
data = {
'level': level,
'owner': {
'name': self.system_name,
'version': None
},
'data': message,
'tags': event.get('tags') or []
}
self._send(data)
def wrapped(event):
try:
# If there is a failure, use the type and message to create the main msg.
if 'failure' in event:
f = event['failure']
parts = [f.type.__name__, f.getErrorMessage()]
msg = ' '.join(filter(None, parts))
# Otherwise use normal event formatting to create the main msg.
else:
msg = formatEvent(event)
new = OrderedDict([
('level', level_name[event.pop('log_level')]),
('msg', msg),
])
if 'log_namespace' in event:
new['namespace'] = event.pop('log_namespace')
if 'log_system' in event:
new['system'] = event.pop('log_system')
# Keep all keys except the noise.
for k, v in sorted(event.items()):
if k not in noisey_keys:
new[k] = v
except Exception as e:
# Fallback to normal event processing
new = event
new['log_failure'] = str(e)
output.write(new['level'].upper() + ' ')
json(new)
def kafka_observer(event):
message = formatEvent(event)
event_dict = dict()
for slot in ['log_logger', 'log_source', 'log_failure']:
if slot in event:
event_dict[slot] = repr(event[slot])
event_dict['log_level'] = event['log_level'].name
for slot in six.iterkeys(event):
if slot not in event_dict:
event_dict[slot] = str(event[slot])
event_dict['klog_level'] = level_mapping[event_dict['log_level']]
event_dict['klog_time'] = event_dict['log_time']
event_dict['klog_message'] = message
json_dump = json.dumps(event_dict)
lock.acquire()
try:
KafkaLogService.producer.send(str(event['log_namespace']).encode('utf-8') + '.json',
json_dump.encode('utf-8'))
KafkaLogService.producer.send('all.json', json_dump.encode('utf-8'))
KafkaLogService.producer.send(str(event['log_namespace']).encode('utf-8') + '.txt',
message.encode('utf-8'))
KafkaLogService.producer.send('all.txt', message.encode('utf-8'))
KafkaLogService.producer.flush()
finally:
lock.release()
def StandardErrorObserver(event):
if event["log_level"] not in levels:
return
if event.get("log_system", u"-") == u"-":
logSystem = u"{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace", event.get("log_namespace", ''))
if event.get("log_format", None) is not None:
eventText = formatEvent(event)
else:
eventText = ""
if "log_failure" in event:
# This is a traceback. Print it.
eventText = eventText + event["log_failure"].getTraceback()
if format == "colour":
# Errors are always red, no matter the system they came from.
eventString = COLOUR_FORMAT.format(
Fore.RED, formatTime(event["log_time"]), logSystem, Fore.RESET,
eventText)
elif format == "nocolour":
eventString = NOCOLOUR_FORMAT.format(
formatTime(event["log_time"]), logSystem, eventText)
elif format == "syslogd":
eventString = SYSLOGD_FORMAT.format(logSystem, eventText)
print(eventString, file=_file)
def __call__(self, event):
"""
Write event to syslog.
"""
# Figure out what the message-text is.
eventText = formatEvent(event)
if eventText is None:
return
# Figure out what syslog parameters we might need to use.
level = event.get("log_level", None)
if level is None:
if 'log_failure' in event:
level = LogLevel.critical
else:
level = LogLevel.info
priority = LOGLEVEL_MAP[level]
facility = int(event.get('log_facility', DEFAULT_FACILITY))
# Break the message up into lines and send them.
lines = eventText.split('\n')
while lines[-1:] == ['']:
lines.pop()
firstLine = True
for line in lines:
if firstLine:
firstLine = False
else:
line = ' ' + line
self.syslog(priority | facility,
'[%s] %s' % (event.get('log_system', '-'), line))
def _render(event):
if event.get("log_system", u"-") == u"-":
logSystem = u"{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace", event.get("log_namespace", ''))
if event.get("log_format", None) is not None:
eventText = formatEvent(event)
else:
eventText = u""
if "log_failure" in event:
# This is a traceback. Print it.
eventText = eventText + event["log_failure"].getTraceback()
eventString = strip_ansi(STANDARD_FORMAT.format(
startcolour=u'', time=formatTime(event["log_time"]),
system=logSystem, endcolour=u'',
text=eventText)) + os.linesep
return eventString
def __call__(self, event):
# it seems if a twisted.logger.Logger() has .failure() called
# on it, the log_format will be None for the traceback after
# "Unhandled error in Deferred" -- perhaps this is a Twisted
# bug?
if event['log_format'] is None:
msg = u'{0} {1}{2}'.format(
formatTime(event["log_time"]),
failure_format_traceback(event['log_failure']),
os.linesep,
)
if self._encode:
msg = msg.encode('utf8')
self._file.write(msg)
else:
# although Logger will already have filtered out unwanted
# levels, bare Logger instances from Twisted code won't have.
if 'log_level' in event and self._acceptable_level(
event['log_level']):
msg = u'{0} {1}{2}'.format(
formatTime(event["log_time"]),
formatEvent(event),
os.linesep,
)
if self._encode:
msg = msg.encode('utf8')
self._file.write(msg)
def formatLogEvent(event, formatTime=formatTime):
eventText = formatEvent(event)
if not eventText:
return None
eventText = eventText.replace(u"\n", u"\n\t")
timeStamp = formatTime(event.get("log_time", None))
system = event.get("log_system", None)
if system is None:
level = event.get("log_level", None)
if level is None:
levelName = u"-"
else:
levelName = level.name
system = u"{namespace}".format(
namespace=event.get("log_namespace", u"-"),
)
else:
try:
system = unicode(system)
except Exception:
system = u"UNFORMATTABLE"
return u"{timeStamp} [{system}] {level}: {event}\n".format(
timeStamp=timeStamp,
system=system.ljust(35),
level=levelName.upper().ljust(8),
event=eventText,
)
def kafka_observer(event):
message = formatEvent(event)
event_dict = dict()
for slot in ['log_logger', 'log_source', 'log_failure']:
if slot in event:
event_dict[slot] = repr(event[slot])
event_dict['log_level'] = event['log_level'].name
for slot in six.iterkeys(event):
def emit(self, event):
is_error = event.get('isError', False)
try:
text = self.format_kv(event)
except Exception, e:
text = 'Parse error: %s. Original message: %s' % \
(repr(e), formatEvent(event))
is_error = True
def emit(self, event):
is_error = event.get('isError', False)
try:
text = self.format_kv(event)
except Exception, e:
text = 'Parse error: %s. Original message: %s' % \
(repr(e), formatEvent(event))
is_error = True
def runtwisted(config=None):
"""
Run the Twisted server.
"""
globalLogBeginner.beginLoggingTo(
[FileLogObserver(sys.stdout, lambda _: formatEvent(_) + "\n")])
threadpool = ThreadPool(maxthreads=30)
app = api.makeapp(config=config)
wsgi_app = WSGIResource(reactor, threadpool, app)
class OptimaResource(Resource):
isLeaf = True
def __init__(self, wsgi):
self._wsgi = wsgi
def render(self, request):
request.prepath = []
request.postpath = ['api'] + request.postpath[:]
r = self._wsgi.render(request)
request.responseHeaders.setRawHeaders(
b'Cache-Control',
[b'no-cache', b'no-store', b'must-revalidate'])
request.responseHeaders.setRawHeaders(b'expires', [b'0'])
return r
# If we have a full path for the client directory, use that directory.
if os.path.isabs(config.CLIENT_DIR):
clientDirTarget = config.CLIENT_DIR
# Otherwise (we have a relative path), use it (correcting so it is with
# respect to the sciris repo directory).
else:
clientDirTarget = '%s%s%s' % (os.pardir, os.sep, config.CLIENT_DIR)
base_resource = File('%s%sdist%s' % (clientDirTarget, os.sep, os.sep))
base_resource.putChild(
'dev', File('%s%ssrc%s' % (clientDirTarget, os.sep, os.sep)))
base_resource.putChild('api', OptimaResource(wsgi_app))
site = Site(base_resource)
try:
port = str(sys.argv[1])
except IndexError:
port = "8091"
# Start the threadpool now, shut it down when we're closing
threadpool.start()
reactor.addSystemEventTrigger('before', 'shutdown', threadpool.stop)
endpoint = serverFromString(reactor, "tcp:port=" + port)
endpoint.listen(site)
reactor.run()
def StandardErrorObserver(event):
if event["log_level"] not in levels:
return
if event.get("log_system", u"-") == u"-":
logSystem = u"{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace",
event.get("log_namespace", ''))
if event.get("log_format", None) is not None:
eventText = formatEvent(event)
else:
eventText = ""
if "log_failure" in event:
# This is a traceback. Print it.
eventText = eventText + event["log_failure"].getTraceback()
if format == "colour":
# Errors are always red, no matter the system they came from.
eventString = COLOUR_FORMAT.format(Fore.RED,
formatTime(event["log_time"]),
logSystem, Fore.RESET,
eventText)
elif format == "nocolour":
eventString = NOCOLOUR_FORMAT.format(formatTime(event["log_time"]),
logSystem, eventText)
elif format == "syslogd":
eventString = SYSLOGD_FORMAT.format(logSystem, eventText)
elif format == "none":
eventString = formatEvent(event)
else:
assert False
if not format == "colour":
eventString = strip_ansi(eventString)
print(eventString, file=_file)
def StandardOutObserver(event):
if event["log_level"] not in levels:
return
if event.get("log_system", "-") == "-":
logSystem = "{:<10} {:>6}".format("Controller", os.getpid())
else:
logSystem = event["log_system"]
if show_source and event.get("log_namespace") is not None:
logSystem += " " + event.get("cb_namespace",
event.get("log_namespace", ''))
if format == "colour":
# Choose a colour depending on where the log came from.
if "Controller" in logSystem:
fore = Fore.BLUE
elif "Router" in logSystem:
fore = Fore.YELLOW
elif "Container" in logSystem:
fore = Fore.GREEN
else:
fore = Fore.WHITE
eventString = COLOUR_FORMAT.format(fore,
formatTime(event["log_time"]),
logSystem, Fore.RESET,
formatEvent(event))
elif format == "nocolour":
eventString = NOCOLOUR_FORMAT.format(formatTime(event["log_time"]),
logSystem, formatEvent(event))
elif format == "syslogd":
eventString = SYSLOGD_FORMAT.format(logSystem, formatEvent(event))
elif format == "none":
eventString = formatEvent(event)
else:
assert False
if not format == "colour":
eventString = strip_ansi(eventString)
print(eventString, file=_file)
def _make_json(_event):
event = dict(_event)
level = event.pop("log_level", LogLevel.info).name
# as soon as possible, we wish to give up if this event is
# outside our target log-level; this is to prevent
# (de-)serializing all the debug() messages (for example) from
# workers to the controller.
if log_levels.index(level) > log_levels.index(get_global_log_level()):
return
done_json = {
"level": level,
"namespace": event.pop("log_namespace", '')
}
eventText = formatEvent(event)
if "log_failure" in event:
# This is a traceback. Print it.
traceback = event["log_failure"].getTraceback()
if not six.PY3:
traceback = traceback.decode('utf-8')
linesep = os.linesep.decode('utf-8')
else:
linesep = os.linesep
eventText = eventText + linesep + traceback
done_json["text"] = escape_formatting(eventText)
try:
event.pop("log_logger", "")
event.pop("log_format", "")
event.pop("log_source", "")
event.pop("log_system", "")
event.pop("log_failure", "")
event.pop("failure", "")
event.update(done_json)
text = encoder.encode(event)
except Exception:
text = encoder.encode({
"text": done_json["text"],
"level": "error",
"namespace": "crossbar._logging"
})
if not isinstance(text, six.text_type):
text = text.decode('utf8')
print(text, end=record_separator, file=outFile)
outFile.flush()
def _log_observer(self, event):
if not self.DEBUG and (event["log_namespace"] != self.log.namespace
or self.log_level > event["log_level"]):
return
msg = formatEvent(event)
click.echo("%s [%s] %s" % (
datetime.fromtimestamp(
event["log_time"]).strftime("%Y-%m-%d %H:%M:%S"),
event["log_level"].name,
msg,
))
def check(self, **features):
for key, expected_value in six.iteritems(features):
if key == 'text':
text = formatEvent(self.event)
if text != expected_value:
raise AssertionError('Expected log event to look like\n{!r}\nbut was\n{!r}\n{!r}'.format(
expected_value, text, self.event))
else:
if self.event[key] != expected_value:
raise AssertionError('Expected log event to have {!r}: {!r} but was \n{!r}'.format(
key, expected_value, self.event))
def check(self, **features):
for key, expected_value in six.iteritems(features):
if key == 'text':
text = formatEvent(self.event)
if text != expected_value:
raise AssertionError('Expected log event to look like\n{!r}\nbut was\n{!r}\n{!r}'.format(
expected_value, text, self.event))
else:
if self.event[key] != expected_value:
raise AssertionError('Expected log event to have {!r}: {!r} but was \n{!r}'.format(
key, expected_value, self.event))
def _make_json(_event):
event = dict(_event)
level = event.pop("log_level", LogLevel.info).name
# as soon as possible, we wish to give up if this event is
# outside our target log-level; this is to prevent
# (de-)serializing all the debug() messages (for example) from
# workers to the controller.
if log_levels.index(level) > log_levels.index(get_global_log_level()):
return
done_json = {
"level": level,
"namespace": event.pop("log_namespace", '')
}
eventText = formatEvent(event)
if "log_failure" in event:
# This is a traceback. Print it.
traceback = event["log_failure"].getTraceback()
if not six.PY3:
traceback = traceback.decode('utf-8')
linesep = os.linesep.decode('utf-8')
else:
linesep = os.linesep
eventText = eventText + linesep + traceback
done_json["text"] = escape_formatting(eventText)
try:
event.pop("log_logger", "")
event.pop("log_format", "")
event.pop("log_source", "")
event.pop("log_system", "")
event.pop("log_failure", "")
event.pop("failure", "")
event.update(done_json)
text = encoder.encode(event)
except Exception:
text = encoder.encode({
"text": done_json["text"],
"level": "error",
"namespace": "crossbar._logging"})
if not isinstance(text, six.text_type):
text = text.decode('utf8')
print(text, end=record_separator, file=outFile)
outFile.flush()
def __call__(self, event):
''' Main dispatcher for log-events '''
# STEP 1) log_system must always be valid. Lumina use it to pass
# on log_namespace mostly. Both the SyslogObserver and
# formatLuminaLogText depend on it being set.
system = event.get("log_system", None)
if system is None:
system = event.get("log_namespace", u"-")
else:
try:
system = STRTYPE(system)
except Exception:
system = u"UNFORMATTABLE"
event['log_system'] = system
# STEP 2) Inject the special lumina logging types
# It works by seraching if any of the special variables are
# present in the event. If it is, the given print function is
# run and the variable is stored in _name. This can then be
# accessed from the logged text by using {_name}.
#fmt = event['log_format']
for (var, fn) in log_specials.items():
if var in event:
event['_' + var] = fn(event[var])
# If the special var is not present in the
# format string, add it to the end
#dvar = '{_' + var + '}'
#if dvar not in fmt:
# fmt += dvar
#event['log_format'] = fmt
# STEP 3) If a log_failure is encountered, fetch the traceback
# and modify log_format to contain the original message +
# the traceback.
if "log_failure" in event:
eventText = formatEvent(event)
try:
traceback = event["log_failure"].getTraceback()
except:
traceback = u"(UNABLE TO OBTAIN TRACEBACK FROM EVENT)\n"
event['log_text'] = u"\n".join((eventText, traceback))
event['log_format'] = '{log_text}'
def _formatModernEvent(event):
"""Format a "modern" event according to MAAS's conventions."""
text = twistedModern.formatEvent(event)
time = event["log_time"] if "log_time" in event else None
level = event["log_level"] if "log_level" in event else None
system = event["log_system"] if "log_system" in event else None
if system is None and "log_namespace" in event:
system = _getSystemName(event["log_namespace"])
return _lineFormat % {
"asctime": twistedModern.formatTime(time, _timeFormat),
"levelname": "-" if level is None else level.name,
"message": "-" if text is None else text.replace("\n", "\n\t"),
"name": "-" if system is None else system,
}
def _make_json(_event):
event = dict(_event)
done_json = {
"level": event.pop("log_level", LogLevel.info).name,
"namespace": event.pop("log_namespace", '')
}
eventText = formatEvent(event)
if "log_failure" in event:
# This is a traceback. Print it.
traceback = event["log_failure"].getTraceback()
if not six.PY3:
traceback = traceback.decode('utf-8')
linesep = os.linesep.decode('utf-8')
else:
linesep = os.linesep
eventText = eventText + linesep + traceback
done_json["text"] = escape_formatting(eventText)
try:
event.pop("log_logger", "")
event.pop("log_format", "")
event.pop("log_source", "")
event.pop("log_system", "")
event.pop("log_failure", "")
event.pop("failure", "")
event.update(done_json)
text = encoder.encode(event)
except Exception:
text = encoder.encode({
"text": done_json["text"],
"level": "error",
"namespace": "crossbar._logging"
})
if not isinstance(text, six.text_type):
text = text.decode('utf8')
print(text, end=record_separator, file=outFile)
outFile.flush()
def catchLogs(testCase, logPublisher=globalLogPublisher):
"""
Catch the global log stream.
@param testCase: The test case to add a cleanup to.
@param logPublisher: the log publisher to add and remove observers for.
@return: a 0-argument callable that returns a list of textual log messages
for comparison.
@rtype: L{list} of L{unicode}
"""
logs = []
logPublisher.addObserver(logs.append)
testCase.addCleanup(lambda: logPublisher.removeObserver(logs.append))
return lambda: [formatEvent(event) for event in logs]
def catchLogs(testCase, logPublisher=globalLogPublisher):
"""
Catch the global log stream.
@param testCase: The test case to add a cleanup to.
@param logPublisher: the log publisher to add and remove observers for.
@return: a 0-argument callable that returns a list of textual log messages
for comparison.
@rtype: L{list} of L{unicode}
"""
logs = []
logPublisher.addObserver(logs.append)
testCase.addCleanup(lambda: logPublisher.removeObserver(logs.append))
return lambda: [formatEvent(event) for event in logs]
def _make_json(_event):
event = dict(_event)
done_json = {
"level": event.pop("log_level", LogLevel.info).name,
"namespace": event.pop("log_namespace", '')
}
eventText = formatEvent(event)
if "log_failure" in event:
# This is a traceback. Print it.
traceback = event["log_failure"].getTraceback()
if not six.PY3:
traceback = traceback.decode('utf-8')
linesep = os.linesep.decode('utf-8')
else:
linesep = os.linesep
eventText = eventText + linesep + traceback
done_json["text"] = escape_formatting(eventText)
try:
event.pop("log_logger", "")
event.pop("log_format", "")
event.pop("log_source", "")
event.pop("log_system", "")
event.pop("log_failure", "")
event.pop("failure", "")
event.update(done_json)
text = encoder.encode(event)
except Exception:
text = encoder.encode({
"text": done_json["text"],
"level": "error",
"namespace": "crossbar._logging"})
if not isinstance(text, six.text_type):
text = text.decode('utf8')
print(text, end=record_separator, file=outFile)
outFile.flush()
def run_app_in_twisted():
globalLogBeginner.beginLoggingTo(
[FileLogObserver(sys.stdout, lambda _: formatEvent(_) + "\n")])
threadpool = ThreadPool(maxthreads=30)
wsgi_app = WSGIResource(reactor, threadpool, app)
class ServerResource(Resource):
isLeaf = True
def __init__(self, wsgi):
Resource.__init__(self)
self._wsgi = wsgi
def render(self, request):
"""
Adds headers to disable caching of api calls
"""
request.prepath = []
request.postpath = ['api'] + request.postpath[:]
r = self._wsgi.render(request)
request.responseHeaders.setRawHeaders(
b'Cache-Control',
[b'no-cache', b'no-store', b'must-revalidate'])
request.responseHeaders.setRawHeaders(b'expires', [b'0'])
return r
# web-client files served from here
base_resource = File('../client/dist')
# api requests must go through /api
base_resource.putChild('api', ServerResource(wsgi_app))
# downloadable files go here
base_resource.putChild('file', File(config.SAVE_FOLDER))
site = Site(base_resource)
# Start the threadpool now, shut it down when we're closing
threadpool.start()
reactor.addSystemEventTrigger('before', 'shutdown', threadpool.stop)
endpoint = serverFromString(reactor, "tcp:port=" + str(config.PORT))
endpoint.listen(site)
reactor.run()
def run():
"""
Run the server.
"""
globalLogBeginner.beginLoggingTo(
[FileLogObserver(sys.stdout, lambda _: formatEvent(_) + "\n")])
threadpool = ThreadPool(maxthreads=30)
wsgi_app = WSGIResource(reactor, threadpool, api.app)
class ScirisResource(Resource):
isLeaf = True
def __init__(self, wsgi):
self._wsgi = wsgi
def render(self, request):
request.prepath = []
request.postpath = ['api'] + request.postpath[:]
r = self._wsgi.render(request)
request.responseHeaders.setRawHeaders(
b'Cache-Control',
[b'no-cache', b'no-store', b'must-revalidate'])
request.responseHeaders.setRawHeaders(b'expires', [b'0'])
return r
base_resource = File('.')
base_resource.putChild('api', ScirisResource(wsgi_app))
site = Site(base_resource)
try:
port = str(sys.argv[1])
except IndexError:
port = "8080"
# Start the threadpool now, shut it down when we're closing
threadpool.start()
reactor.addSystemEventTrigger('before', 'shutdown', threadpool.stop)
endpoint = serverFromString(reactor, "tcp:port=" + port)
endpoint.listen(site)
reactor.run()
def run_twisted(port=8080,
flask_app=None,
client_dir=None,
do_log=False,
reactor_args=None):
# Give an error if we pass in no Flask server or client path.
if reactor_args is None:
reactor_args = {}
if (flask_app is None) and (client_dir is None):
print('ERROR: Neither client or server are defined.')
return None
if do_log: # Set up logging.
globalLogBeginner.beginLoggingTo([
FileLogObserver(sys.stdout,
lambda _: formatEvent(_) + "\n")
])
if client_dir is not None: # If there is a client path, set up the base resource.
base_resource = File(client_dir)
# If we have a flask app...
if flask_app is not None:
thread_pool = ThreadPool(
maxthreads=30) # Create a thread pool to use with the app.
wsgi_app = WSGIResource(
reactor, thread_pool, flask_app
) # Create the WSGIResource object for the flask server.
if client_dir is None: # If we have no client path, set the WSGI app to be the base resource.
base_resource = ScirisResource(wsgi_app)
else: # Otherwise, make the Flask app a child resource.
base_resource.putChild(b'api', ScirisResource(wsgi_app))
thread_pool.start(
) # Start the threadpool now, shut it down when we're closing
reactor.addSystemEventTrigger('before', 'shutdown',
thread_pool.stop)
# Create the site.
site = Site(base_resource)
endpoint = serverFromString(
reactor, "tcp:port=" + str(port)
) # Create the endpoint we want to listen on, and point it to the site.
endpoint.listen(site)
reactor.run(**reactor_args) # Start the reactor.
return None
def formatForSystemd(event):
# Events generated by twisted.python.log have a "system", while ones
# generated with twisted.logger have a "namespace" with similar
# meaning.
#
s = "[{}] ".format(event.get("log_system") or event.get("log_namespace") or "-")
s += formatEvent(event)
if not s:
return None
if "log_failure" in event:
try:
s += "\n" + event["log_failure"].getTraceback().rstrip("\n")
except: # noqa
pass
prefix = _txLevelToPriority.get(event.get("log_level")) or "<6>"
return prefix + s.replace("\n", "\n" + prefix + " ") + "\n"
def _make_json(_event):
event = dict(_event)
done_json = {
"level": event.pop("log_level", LogLevel.info).name,
"namespace": event.pop("log_namespace", '')
}
eventText = formatEvent(event)
if "log_failure" in event:
# This is a traceback. Print it.
eventText = eventText + os.linesep + event[
"log_failure"].getTraceback()
done_json["text"] = escape_formatting(eventText)
try:
event.pop("log_logger", "")
event.pop("log_format", "")
event.pop("log_source", "")
event.pop("log_system", "")
event.pop("log_failure", "")
event.pop("failure", "")
event.update(done_json)
text = json.dumps(event, skipkeys=True)
except Exception as e:
text = json.dumps({
"text":
"Error writing: " + str(e) + " " + str(event),
"level":
"error",
"namespace":
"crossbar._logging"
})
if not isinstance(text, six.text_type):
text = text.decode('utf8')
print(text, end=record_separator, file=outFile)
def json_format(self, event):
error = bool(event.get("isError")) or "log_failure" in event
ts = event["log_time"]
if error:
severity = 3
else:
severity = 5
def to_fields(kv):
reply = dict()
for k, v in kv:
if (k not in IGNORED_KEYS and
type(v) in (str, unicode, list, int, float)):
reply[k] = v
return reply
msg = {
"Hostname": HOSTNAME,
"Timestamp": ts * 1000 * 1000 * 1000,
"Type": "twisted:log",
"Severity": event.get("severity") or severity,
"EnvVersion": "2.0",
"Fields": to_fields(event.iteritems()),
"Logger": self.logger_name,
}
# flatten the client_info into Fields
ci = event.get('client_info')
if ci and isinstance(ci, dict):
msg['Fields'].update(
to_fields(ci.iteritems()))
# flatten timings into Fields
ti = event.get('timings')
if ti and isinstance(ti, dict):
msg["Fields"].update(
to_fields(ti.iteritems())
)
# Add the nicely formatted message
msg["Fields"]["message"] = formatEvent(event)
return json.dumps(msg, skipkeys=True) + "\n"
def formatForSystemd(event):
# Events generated by twisted.python.log have a "system", while ones
# generated with twisted.logger have a "namespace" with similar
# meaning.
#
s = "[{}] ".format(event.get("log_system") or
event.get("log_namespace") or "-")
s += formatEvent(event)
if not s:
return None
if "log_failure" in event:
try:
s += "\n" + event["log_failure"].getTraceback().rstrip("\n")
except: # noqa
pass
prefix = _txLevelToPriority.get(event.get("log_level")) or "<6>"
return prefix + s.replace("\n", "\n" + prefix + " ") + "\n"
def _formatModernEvent(event):
"""Format a "modern" event according to MAAS's conventions."""
text = twistedModern.formatEvent(event)
if "log_failure" in event:
try:
traceback = event["log_failure"].getTraceback()
except Exception:
traceback = u"(UNABLE TO OBTAIN TRACEBACK FROM EVENT)\n"
text = "\n".join((text, traceback))
time = event["log_time"] if "log_time" in event else None
level = event["log_level"] if "log_level" in event else None
system = event["log_system"] if "log_system" in event else None
if system is None and "log_namespace" in event:
system = _getSystemName(event["log_namespace"])
return _lineFormat % {
"asctime": twistedModern.formatTime(time, _timeFormat),
"levelname": "-" if level is None else level.name,
"message": "-" if text is None else text.replace("\n", "\n\t"),
"name": "-" if system is None else system,
}
def format(self, record):
# Create message dict
message = {
'@timestamp': self.format_timestamp(record['log_time']),
'@version': self.version,
'message': logger.formatEvent(record),
'host': self.host,
'path': record['log_stack'][-1][1],
'function': record['log_stack'][-1][3],
'tags': self.tags,
'type': self.message_type,
'levelname': record['log_level'].name,
'logger': self.get_namespace(record),
}
# extra fields
message.update(self.get_extra_fields(record))
# exception infos
if 'log_failure' in record:
message.update(self.get_debug_fields(record))
return self.serialize(message)
def format_event(event):
"""
Formats the given event into text output that just contains the
logging level and message (no timestamp or namespace).
Args:
event (dict): Dict representing the emitted log event
Returns:
unicode: Unicode string to be printed in the logs
"""
level = event.get("log_level", None)
level_name = level.name if level is not None else "-"
log_level = "[{level_name}]".format(level_name=level_name)
event_text = formatEvent(event)
event_text = event_text.replace("\n", "\n\t")
# Pad the level so that regardless of logging level the left
# edge of the event text is aligned.
return "{level: <7} {event_text}\n".format(level=log_level,
event_text=event_text)
def formatLuminaLogText(event, formatTime=formatTime):
""" Format an event as a line of human-readable text. """
eventText = formatEvent(event)
if not eventText:
return None
eventText = eventText.replace(u"\n", u"\n\t")
timeStamp = formatTime(event.get("log_time", None))
system = event.get("log_system", None)
level = event.get("log_level", None)
if level is None:
levelName = u"-"
else:
levelName = level.name
return u"{timeStamp} {level:<8} [{system}] {event}\n".format(
timeStamp=timeStamp,
level=levelName,
system=system,
event=eventText,
)
def _make_json(_event):
event = dict(_event)
done_json = {
"level": event.pop("log_level", LogLevel.info).name,
"namespace": event.pop("log_namespace", '')
}
eventText = formatEvent(event)
if "log_failure" in event:
# This is a traceback. Print it.
eventText = eventText + os.linesep + event["log_failure"].getTraceback()
done_json["text"] = escape_formatting(eventText)
try:
event.pop("log_logger", "")
event.pop("log_format", "")
event.pop("log_source", "")
event.pop("log_system", "")
event.pop("log_failure", "")
event.pop("failure", "")
event.update(done_json)
text = json.dumps(event, skipkeys=True)
except Exception as e:
text = json.dumps({"text": "Error writing: " + str(e) + " " + str(event),
"level": "error",
"namespace": "crossbar._logging"})
if not isinstance(text, six.text_type):
text = text.decode('utf8')
print(text, end=record_separator, file=outFile)
def json_format(self, event):
error = bool(event.get("isError")) or "failure" in event
ts = event["log_time"]
if error:
severity = 3
else:
severity = 5
msg = {
"Hostname": HOSTNAME,
"Timestamp": ts * 1000 * 1000 * 1000,
"Type": "twisted:log",
"Severity": event.get("severity") or severity,
"EnvVersion": "2.0",
"Fields": {k: v for k, v in event.iteritems()
if k not in IGNORED_KEYS and
type(v) in (str, unicode, list, int, float)},
"Logger": self.logger_name,
}
# Add the nicely formatted message
msg["Fields"]["message"] = formatEvent(event)
return json.dumps(msg, skipkeys=True) + "\n"
def _make_json(event):
return json.dumps({"text": formatEvent(event).replace(u"{", u"{{").replace(u"}", u"}}"),
"level": event.get("log_level", LogLevel.info).name})
def simpleObserver(event):
print(formatEvent(event))
def printing_observer(event): print formatEvent(event)
def render(events=self.events):
for event in events:
rendered = formatEvent(event)
if rendered is not None:
yield rendered.encode("utf-8")
bcolor = {'debug':'\033[94m',
'info':'\033[92m',
'warn':'\033[93m',
'error':'\033[91m',
'default':'\033[33m',
}
@provider(ILogObserver)
def simpleObserver(event):
# event['log_system'] = "asdf"
print event
print(formatEvent(event))
logFormat = lambda event: u"{0} [{1}]: {2}".format(formatTime(event["log_time"]), event["log_level"].name.upper(),
formatEvent(event))
@provider(ILogObserver)
def consoleLogObserver(event):
print u"[{0}{1}\033[39m-{2}]: {3}".format(bcolor[event["log_level"].name.lower()], event["log_level"].name.upper(), event["log_namespace"], formatEvent(event))
def get_logger(logname='yombolog', **kwargs):
"""
Returns a logger object that allows logging of error messages.
**Usage**:
.. code-block:: python
from yombo.core.log import get_logger
def simpleObserver(event):
print(formatEvent(event))
def simpleObserver(event):
# event['log_system'] = "asdf"
print event
print(formatEvent(event))
def consoleLogObserver(event):
print u"[{0}{1}\033[39m-{2}]: {3}".format(bcolor[event["log_level"].name.lower()], event["log_level"].name.upper(), event["log_namespace"], formatEvent(event))
def _make_json(event):
return json.dumps({
"text": escape_formatting(formatEvent(event)),
"level": event.get("log_level", LogLevel.info).name})
