def __init__(self, name: str, *,
java: os_path.isfile,
arguments: tuple_of(str),
classpath: str,
jndi: dict_of(str, str),
factory: str,
queue: str,
username: str,
password: str,
request_timeout: optional(float) = None,
**kwargs): # this kwargs allows for extra application-specific
# settings in config_interface_jms_X.py
self._name = name
AdapterHost.__init__(self, "org.pythomnic.jms.Receiver",
java = java, arguments = arguments, classpath = classpath,
jndi = jndi, factory = factory, queue = queue,
username = username, password = password)
self._request_timeout = request_timeout or \
pmnc.config_interfaces.get("request_timeout") # this is now static
if pmnc.request.self_test == __name__: # self-test
self._process_request = kwargs["process_request"]
def __init__(self, name: str, *,
java: os_path.isfile,
arguments: tuple_of(str),
classpath: str,
jndi: dict_of(str, str),
factory: str,
queue: str,
username: str,
password: str):
TransactionalResource.__init__(self, name)
AdapterHost.__init__(self, "org.pythomnic.jms.Sender",
java = java, arguments = arguments, classpath = classpath,
jndi = jndi, factory = factory, queue = queue,
username = username, password = password)
self._request_count = 0
def __init__(self, class_name: str, *,
java: os_path.isfile,
arguments: tuple_of(str),
classpath: str,
jndi: dict_of(str, str),
factory: str,
queue: str,
username: str,
password: str):
# random line prefix and suffix for packet serialization
bol = b2a_hex(urandom(8)).decode("ascii").upper()
eol = b2a_hex(urandom(8)).decode("ascii").upper()
# compose the executable command line
self._args = [ java ] + list(arguments) + \
[ "-classpath", classpath, class_name,
"connection.factory={0:s}".format(factory),
"connection.queue={0:s}".format(queue),
"stdout.bol={0:s}".format(bol),
"stdout.eol={0:s}".format(eol) ]
if username or password:
self._args.append("connection.username={0:s}".format(username))
self._args.append("connection.password={0:s}".format(password))
self._args.extend("jndi.{0:s}={1:s}".format(*t) for t in jndi.items())
self._bol_b = bol.encode("ascii")
self._eol_b = eol.encode("ascii")
# this set tracks messages that have been processed but
# not committed on the server due to a failure
self._processed_messages = set()
def _notifications_report(html: with_attr("write"), query: dict_of(str, str),
request: dict, response: dict) -> nothing:
# reassemble the canonical URL query
canonical_query = "&".join("{0:s}={1:s}".format(k, v) for k, v in query.items())
# format header
cage_at_node = "cage {0:s} at node {1:s}".format(__cage__, __node__)
# write page header
html.write("<html>"
"<head>" +
css_style.format(css_font_family = pmnc.config.get("css_font_family")) +
"<title>Notifications report for " + cage_at_node + "</title>"
"</head>"
"<body class=\"default\">"
"<a href=\"/performance\">perf</a>" + _decorate(" {0:s}<br/>".format(cage_at_node.center(58))) +
_decorate("most recent health monitor notifications".center(69)) + "<br/><br/>")
# extract stored notifications and loop through it
last_day = None
for notification in reversed(pmnc.notify.extract()):
message = _quote(notification["message"])
timestamp = notification["timestamp"]
day = datetime.fromtimestamp(timestamp).strftime("%b %d")
hms = datetime.fromtimestamp(timestamp).strftime("%H:%M:%S")
level = notification["level"]
if level == "INFO":
level = "<span class=\"c4\">INFO </span>"
elif level == "WARNING":
level = "<span class=\"c1\">WARN </span>"
elif level == "ERROR":
level = "<span class=\"c0\">ERROR</span>"
elif level == "ALERT":
level = "<strong><span class=\"c0\">ALERT</span></strong>"
if day != last_day:
html.write(_decorate(" {0:s} ----------------------------------------------------------------<br/>\n".format(day)))
last_day = day
html.write("{0:s} {1:s} <nobr>{2:s}</nobr><br/>\n".\
format(hms, level, message))
# complete the response
html.write(_decorate("------------------------------------------------------------------------<br/>\n"))
html.write("</body></html>")
# require a refresh within a configured time
refresh_seconds = pmnc.config.get("refresh_seconds")
response["headers"]["refresh"] = \
"{0:d};URL=/notifications?{1:s}".format(refresh_seconds, canonical_query)
def _performance_report(html: with_attr("write"), query: dict_of(str, str),
request: dict, response: dict) -> nothing:
# fetch the performance dump, this can return None
stats = pmnc.performance.extract()
if stats is None:
html.write("<html><body>cage {0:s} has nothing to report yet"
"</body></html>".format(__cage__))
return
# extract main thread pool stats
req_active, req_pending, req_rate = pmnc.interfaces.get_activity_stats()
# extract global transaction rate
xa = pmnc.transaction.create(); xa.execute()
txn_rate = xa.get_transaction_rate()
base_time, stats_dump, app_perf = stats
base_dt = datetime.fromtimestamp(base_time)
# see data for what objects is available, will have to display at least one graph for each
monitored_objects = set(tuple(k.split(".", 2)[:2]) for k in stats_dump.keys())
requested_objects = set(tuple(k.split(".", 2)[:2]) for k in query.keys())
displayed_objects = {}
for object_type, object_name in monitored_objects - requested_objects:
reading, mode = default_reading_modes[object_type]
displayed_objects["{0:s}.{1:s}.{2:s}".format(object_type, object_name, reading)] = mode
# add/override explicitly requested graphs from the URL query
for k, v in query.items():
if k in stats_dump:
displayed_objects[k] = v
# reassemble the canonical URL query
canonical_query = _format_modified_query(displayed_objects)
# format horizontal time scales
base_minute = base_dt.minute % 10
hrule = "---- " + _hrule[base_minute:][:59] + " -----"
plus_count = hrule.count("+")
base_dt10m = datetime.fromtimestamp(base_time // 600 * 600)
base_dt10m -= timedelta(minutes = (plus_count - (base_minute != 0 and 1 or 0)) * 10)
hhmms = [ (base_dt10m + timedelta(minutes = i * 10)).strftime("%H:%M")
for i in range(plus_count) ]
hscale = " " * (hrule.index("+") - 2) + " ".join(hhmms)
hscale += " " * (len(hrule) - len(hscale) - 5) + base_dt.strftime("%H:%M")
hrule = _decorate(hrule)
hscale = _decorate(hscale)
# format header
cpu_ut_percent = int(app_perf.get("cpu_ut_percent", 0.0))
cpu_kt_percent = int(app_perf.get("cpu_kt_percent", 0.0))
if cpu_kt_percent > 0:
cpu_percent = "{0:d}*{1:d}".format(cpu_ut_percent, cpu_kt_percent)
else:
cpu_percent = "{0:d}".format(cpu_ut_percent)
cage_at_node = "cage {0:s} at node {1:s}".format(__cage__, __node__)
activity_info = "{0:d}{1:s} req, {2:.01f} req/s, {3:.01f} txn/s, {4:d} M RAM, {5:s} % CPU".\
format(req_active, req_pending and "*{0:d}".format(req_pending) or "",
req_rate, txn_rate, app_perf.get("wss", 0), cpu_percent)
# write page header
html.write("<html>"
"<head>" +
css_style.format(css_font_family = pmnc.config.get("css_font_family")) +
"<title>Performance report for " + cage_at_node + "</title>"
"</head>"
"<body class=\"default\">"
"<a href=\"/notifications\">logs</a>" + _decorate(" {0:s}<br/>".format(cage_at_node.center(58))) +
_decorate(" {0:s} {1:s}<br/><br/>\n".format(activity_info.center(58), base_dt.strftime("%b %d"))))
html.write("<span style=\"line-height: 1.0;\">\n" + hscale + "<br/>\n" + hrule + "<br/>\n")
# loop through the statistics items to display
for k, mode in sorted(displayed_objects.items()):
if k not in stats_dump: # invalid interface/resource name in URL ?
continue
s1m, s10s = stats_dump[k]
s10s = s10s[:5]
object_type, object_name, reading = k.split(".", 2)
if mode == "collapsed": # draw collapsed graph
def append_bars(b):
result = ""
for i, v in enumerate(b):
c = _collapsed_bar(v)
high = v is not None and v[1] or 0
result += "<span_class=\"c{0:d}\">{1:s}</span>".format(4 - high // 8, c)
return result
line = "<nobr> {0:s} {1:s} ".format(append_bars(s1m), append_bars(s10s))
html.write(_decorate(line))
# append the clickable expand link
expand_query = _format_modified_query(displayed_objects, expand = k)
html.write("<a href=\"/performance?{0:s}\">{1:s} {2:s} ({3:s})</a></nobr><br/>".\
format(expand_query, object_type, object_name, legends[reading]))
elif mode == "expanded": # draw expanded graph
bars1m = list(zip(*map(_expanded_bar, s1m)))
bars10s = list(zip(*map(_expanded_bar, s10s)))
def append_bars(b, i):
result = ""
b = b[4 - i]
for j in range(len(b)):
c = b[j]
if c.startswith("~"):
result += "<span_class=\"c{0:d}i\">{1:s}</span>".format(i, c[1:]) # note the underscore, will be converted to space
else:
result += c
return result
# expanded form has vertical scale, different for rates and times
if "_time" in k:
ref = _ref_times
elif "_rate" in k:
ref = _ref_rates
# draw each of the 5 horizontal bars
opt_readings = optional_readings[object_type]
for i in range(5):
line = "<nobr>" + ref[i] + "<span class=\"c{0:d}\">".format(i)
_line = append_bars(bars1m, i)
_line += "</span> ~ <span_class=\"c{0:d}\">".format(i) + append_bars(bars10s, i) + " "
line += _decorate(_line) + "</span>"
if i == 0: # append the clickable collapse link
collapse_query = _format_modified_query(displayed_objects, collapse = k)
line += "<a href=\"/performance?{0:s}\">{1:s} {2:s}</a>".\
format(collapse_query, object_type, object_name)
elif i <= len(opt_readings): # append the clickable selector links
opt_reading = opt_readings[i - 1]
modify_query = _format_modified_query(displayed_objects, replace = (k, opt_reading))
line += "{0:s}<a href=\"/performance?{1:s}\">{2:s}</a>{3:s}".\
format(reading == opt_reading and _decorate("» ") or _decorate(" "),
modify_query, legends[opt_reading],
reading == opt_reading and _decorate(" «") or _decorate(" "))
html.write(line + "</nobr><br/>\n")
html.write(hrule + "<br/>\n")
# complete the response
html.write(hscale + "<br/>\n</span>\n</body></html>")
# require a refresh within a configured time
refresh_seconds = pmnc.config.get("refresh_seconds")
response["headers"]["refresh"] = \
"{0:d};URL=/performance?{1:s}".format(refresh_seconds, canonical_query)
def foo(*, k: tc.dict_of((int, int), [tc.has("^[0-9]+$"), tc.has("^[0-9]+$")])):
return functools.reduce(lambda r, t: r and str(t[0][0]) == t[1][0] and str(t[0][1]) == t[1][1],
k.items(), True)
def foo(x: tc.dict_of(int, str)) -> tc.dict_of(str, int):
return { v: k for k, v in x.items() }
assert foo(k = { (1, 2): ["1", "2"], (3, 4): ["3", "4"]})
assert not foo(k = { (1, 3): ["1", "2"], (3, 4): ["3", "4"]})
assert not foo(k = { (1, 2): ["1", "2"], (3, 4): ["3", "5"]})
with expected(InputParameterError("foo() has got an incompatible value for k: {(1, 2): ['1', '2'], (3, 4): ['3', 'x']}")):
foo(k = { (1, 2): ["1", "2"], (3, 4): ["3", "x"]})
with expected(InputParameterError("foo() has got an incompatible value for k: {(1, 2): ['1', '2'], (3,): ['3', '4']}")):
foo(k = { (1, 2): ["1", "2"], (3, ): ["3", "4"]})
with expected(InputParameterError("foo() has got an incompatible value for k: {(1, 2): ['1', '2'], (3, 4.0): ['3', '4']}")):
foo(k = { (1, 2): ["1", "2"], (3, 4.0): ["3", "4"]})
###################
assert tc.dict_of(int, tc.optional(str))({ 1: "foo", 2: None }) and \
tc.dict_of(int, tc.optional(str)).check({ 1: "foo", 2: None })
assert not tc.dict_of(int, tc.optional(str))({ None: "foo", 2: None }) and \
not tc.dict_of(int, tc.optional(str)).check({ None: "foo", 2: None })
print("ok")
############################################################################
print("enum: ", end="")
@typecheck
def foo(x: tc.enum(int, 1)) -> tc.enum(1, int):
return x
class Packet(dict):
_valid_key = by_regex("^[A-Za-z0-9_]+$")
_valid_keys = dict_of(_valid_key, str)
_bol_b = b"4F36095410830A13"
_eol_b = b"92B4782E3B570FD3"
@typecheck
def __init__(self, **params):
assert self._valid_keys(params)
self.update(params)
###################################
@classmethod
@typecheck
def load_from_stream(cls, stream: with_attr("readline"), bol_b: bytes, eol_b: bytes):
params = {}
def decode_line(b):
key, value = b.split(b"=", 1)
key, value = key.decode("ascii"), a2b_base64(value).decode("utf-8")
params[key] = value
# the output lines may contain garbage emitted by any java
# library and they are therefore filtered and checksummed
valid_line = regex(b"^.*(?:" + bol_b + b"|" + cls._bol_b + b")" +
b"([0-9A-F]{8})( ?[A-Za-z0-9+/=_]*)" +
b"(?:" + eol_b + b"|" + cls._eol_b + b").*$")
def get_next_line(prev_crc32 = 0):
while True:
b = stream.readline()
if not b:
return None
b = b.rstrip()
if not b:
continue
valid_parts = valid_line.findall(b)
if len(valid_parts) != 1:
pmnc.log.warning("skipping unexpected output: {0:s}".format(str(b)[2:-1]))
continue
next_crc32, bb = int(valid_parts[0][0], 16), valid_parts[0][1]
if next_crc32 != crc32(bb, prev_crc32):
pmnc.log.warning("skipping broken output: {0:s}".format(str(b)[2:-1]))
continue
return bb, next_crc32
curr_lines = []
next_line_crc32 = get_next_line()
if next_line_crc32 is None:
return None
next_line, curr_crc32 = next_line_crc32
while next_line:
if next_line.startswith(b" "):
if curr_lines:
curr_lines.append(next_line[1:])
else:
raise Exception("invalid folding")
else:
if curr_lines:
decode_line(b"".join(curr_lines))
del curr_lines[:]
curr_lines.append(next_line)
next_line_crc32 = get_next_line(curr_crc32)
if next_line_crc32 is None:
raise Exception("unexpected eof")
next_line, curr_crc32 = next_line_crc32
if curr_lines:
decode_line(b"".join(curr_lines))
return cls(**params)
###################################
@typecheck
def save_to_stream(self, stream: with_attr("write", "flush"), fold_width: int):
for k, v in self.items():
encoded = k.encode("ascii") + b"=" + b2a_base64(v.encode("utf-8")).rstrip()
first_line, encoded = encoded[:fold_width], encoded[fold_width:]
stream.write(first_line + b"\n")
for folded_line in [ b" " + encoded[i:i+fold_width-1]
for i in range(0, len(encoded), fold_width-1) ]:
stream.write(folded_line + b"\n")
stream.write(b"\n")
stream.flush()
