def __init__(self, name: str, *, java: os_path.isfile, arguments: tuple_of(str), classpath: str, jndi: dict_of(str, str), factory: str, queue: str, username: str, password: str, request_timeout: optional(float) = None, **kwargs): # this kwargs allows for extra application-specific # settings in config_interface_jms_X.py self._name = name AdapterHost.__init__(self, "org.pythomnic.jms.Receiver", java = java, arguments = arguments, classpath = classpath, jndi = jndi, factory = factory, queue = queue, username = username, password = password) self._request_timeout = request_timeout or \ pmnc.config_interfaces.get("request_timeout") # this is now static if pmnc.request.self_test == __name__: # self-test self._process_request = kwargs["process_request"]
def __init__(self, name: str, *, java: os_path.isfile, arguments: tuple_of(str), classpath: str, jndi: dict_of(str, str), factory: str, queue: str, username: str, password: str): TransactionalResource.__init__(self, name) AdapterHost.__init__(self, "org.pythomnic.jms.Sender", java = java, arguments = arguments, classpath = classpath, jndi = jndi, factory = factory, queue = queue, username = username, password = password) self._request_count = 0
def __init__(self, class_name: str, *, java: os_path.isfile, arguments: tuple_of(str), classpath: str, jndi: dict_of(str, str), factory: str, queue: str, username: str, password: str): # random line prefix and suffix for packet serialization bol = b2a_hex(urandom(8)).decode("ascii").upper() eol = b2a_hex(urandom(8)).decode("ascii").upper() # compose the executable command line self._args = [ java ] + list(arguments) + \ [ "-classpath", classpath, class_name, "connection.factory={0:s}".format(factory), "connection.queue={0:s}".format(queue), "stdout.bol={0:s}".format(bol), "stdout.eol={0:s}".format(eol) ] if username or password: self._args.append("connection.username={0:s}".format(username)) self._args.append("connection.password={0:s}".format(password)) self._args.extend("jndi.{0:s}={1:s}".format(*t) for t in jndi.items()) self._bol_b = bol.encode("ascii") self._eol_b = eol.encode("ascii") # this set tracks messages that have been processed but # not committed on the server due to a failure self._processed_messages = set()
def _notifications_report(html: with_attr("write"), query: dict_of(str, str), request: dict, response: dict) -> nothing: # reassemble the canonical URL query canonical_query = "&".join("{0:s}={1:s}".format(k, v) for k, v in query.items()) # format header cage_at_node = "cage {0:s} at node {1:s}".format(__cage__, __node__) # write page header html.write("<html>" "<head>" + css_style.format(css_font_family = pmnc.config.get("css_font_family")) + "<title>Notifications report for " + cage_at_node + "</title>" "</head>" "<body class=\"default\">" "<a href=\"/performance\">perf</a>" + _decorate(" {0:s}<br/>".format(cage_at_node.center(58))) + _decorate("most recent health monitor notifications".center(69)) + "<br/><br/>") # extract stored notifications and loop through it last_day = None for notification in reversed(pmnc.notify.extract()): message = _quote(notification["message"]) timestamp = notification["timestamp"] day = datetime.fromtimestamp(timestamp).strftime("%b %d") hms = datetime.fromtimestamp(timestamp).strftime("%H:%M:%S") level = notification["level"] if level == "INFO": level = "<span class=\"c4\">INFO </span>" elif level == "WARNING": level = "<span class=\"c1\">WARN </span>" elif level == "ERROR": level = "<span class=\"c0\">ERROR</span>" elif level == "ALERT": level = "<strong><span class=\"c0\">ALERT</span></strong>" if day != last_day: html.write(_decorate(" {0:s} ----------------------------------------------------------------<br/>\n".format(day))) last_day = day html.write("{0:s} {1:s} <nobr>{2:s}</nobr><br/>\n".\ format(hms, level, message)) # complete the response html.write(_decorate("------------------------------------------------------------------------<br/>\n")) html.write("</body></html>") # require a refresh within a configured time refresh_seconds = pmnc.config.get("refresh_seconds") response["headers"]["refresh"] = \ "{0:d};URL=/notifications?{1:s}".format(refresh_seconds, canonical_query)
def _performance_report(html: with_attr("write"), query: dict_of(str, str), request: dict, response: dict) -> nothing: # fetch the performance dump, this can return None stats = pmnc.performance.extract() if stats is None: html.write("<html><body>cage {0:s} has nothing to report yet" "</body></html>".format(__cage__)) return # extract main thread pool stats req_active, req_pending, req_rate = pmnc.interfaces.get_activity_stats() # extract global transaction rate xa = pmnc.transaction.create(); xa.execute() txn_rate = xa.get_transaction_rate() base_time, stats_dump, app_perf = stats base_dt = datetime.fromtimestamp(base_time) # see data for what objects is available, will have to display at least one graph for each monitored_objects = set(tuple(k.split(".", 2)[:2]) for k in stats_dump.keys()) requested_objects = set(tuple(k.split(".", 2)[:2]) for k in query.keys()) displayed_objects = {} for object_type, object_name in monitored_objects - requested_objects: reading, mode = default_reading_modes[object_type] displayed_objects["{0:s}.{1:s}.{2:s}".format(object_type, object_name, reading)] = mode # add/override explicitly requested graphs from the URL query for k, v in query.items(): if k in stats_dump: displayed_objects[k] = v # reassemble the canonical URL query canonical_query = _format_modified_query(displayed_objects) # format horizontal time scales base_minute = base_dt.minute % 10 hrule = "---- " + _hrule[base_minute:][:59] + " -----" plus_count = hrule.count("+") base_dt10m = datetime.fromtimestamp(base_time // 600 * 600) base_dt10m -= timedelta(minutes = (plus_count - (base_minute != 0 and 1 or 0)) * 10) hhmms = [ (base_dt10m + timedelta(minutes = i * 10)).strftime("%H:%M") for i in range(plus_count) ] hscale = " " * (hrule.index("+") - 2) + " ".join(hhmms) hscale += " " * (len(hrule) - len(hscale) - 5) + base_dt.strftime("%H:%M") hrule = _decorate(hrule) hscale = _decorate(hscale) # format header cpu_ut_percent = int(app_perf.get("cpu_ut_percent", 0.0)) cpu_kt_percent = int(app_perf.get("cpu_kt_percent", 0.0)) if cpu_kt_percent > 0: cpu_percent = "{0:d}*{1:d}".format(cpu_ut_percent, cpu_kt_percent) else: cpu_percent = "{0:d}".format(cpu_ut_percent) cage_at_node = "cage {0:s} at node {1:s}".format(__cage__, __node__) activity_info = "{0:d}{1:s} req, {2:.01f} req/s, {3:.01f} txn/s, {4:d} M RAM, {5:s} % CPU".\ format(req_active, req_pending and "*{0:d}".format(req_pending) or "", req_rate, txn_rate, app_perf.get("wss", 0), cpu_percent) # write page header html.write("<html>" "<head>" + css_style.format(css_font_family = pmnc.config.get("css_font_family")) + "<title>Performance report for " + cage_at_node + "</title>" "</head>" "<body class=\"default\">" "<a href=\"/notifications\">logs</a>" + _decorate(" {0:s}<br/>".format(cage_at_node.center(58))) + _decorate(" {0:s} {1:s}<br/><br/>\n".format(activity_info.center(58), base_dt.strftime("%b %d")))) html.write("<span style=\"line-height: 1.0;\">\n" + hscale + "<br/>\n" + hrule + "<br/>\n") # loop through the statistics items to display for k, mode in sorted(displayed_objects.items()): if k not in stats_dump: # invalid interface/resource name in URL ? continue s1m, s10s = stats_dump[k] s10s = s10s[:5] object_type, object_name, reading = k.split(".", 2) if mode == "collapsed": # draw collapsed graph def append_bars(b): result = "" for i, v in enumerate(b): c = _collapsed_bar(v) high = v is not None and v[1] or 0 result += "<span_class=\"c{0:d}\">{1:s}</span>".format(4 - high // 8, c) return result line = "<nobr> {0:s} {1:s} ".format(append_bars(s1m), append_bars(s10s)) html.write(_decorate(line)) # append the clickable expand link expand_query = _format_modified_query(displayed_objects, expand = k) html.write("<a href=\"/performance?{0:s}\">{1:s} {2:s} ({3:s})</a></nobr><br/>".\ format(expand_query, object_type, object_name, legends[reading])) elif mode == "expanded": # draw expanded graph bars1m = list(zip(*map(_expanded_bar, s1m))) bars10s = list(zip(*map(_expanded_bar, s10s))) def append_bars(b, i): result = "" b = b[4 - i] for j in range(len(b)): c = b[j] if c.startswith("~"): result += "<span_class=\"c{0:d}i\">{1:s}</span>".format(i, c[1:]) # note the underscore, will be converted to space else: result += c return result # expanded form has vertical scale, different for rates and times if "_time" in k: ref = _ref_times elif "_rate" in k: ref = _ref_rates # draw each of the 5 horizontal bars opt_readings = optional_readings[object_type] for i in range(5): line = "<nobr>" + ref[i] + "<span class=\"c{0:d}\">".format(i) _line = append_bars(bars1m, i) _line += "</span> ~ <span_class=\"c{0:d}\">".format(i) + append_bars(bars10s, i) + " " line += _decorate(_line) + "</span>" if i == 0: # append the clickable collapse link collapse_query = _format_modified_query(displayed_objects, collapse = k) line += "<a href=\"/performance?{0:s}\">{1:s} {2:s}</a>".\ format(collapse_query, object_type, object_name) elif i <= len(opt_readings): # append the clickable selector links opt_reading = opt_readings[i - 1] modify_query = _format_modified_query(displayed_objects, replace = (k, opt_reading)) line += "{0:s}<a href=\"/performance?{1:s}\">{2:s}</a>{3:s}".\ format(reading == opt_reading and _decorate("» ") or _decorate(" "), modify_query, legends[opt_reading], reading == opt_reading and _decorate(" «") or _decorate(" ")) html.write(line + "</nobr><br/>\n") html.write(hrule + "<br/>\n") # complete the response html.write(hscale + "<br/>\n</span>\n</body></html>") # require a refresh within a configured time refresh_seconds = pmnc.config.get("refresh_seconds") response["headers"]["refresh"] = \ "{0:d};URL=/performance?{1:s}".format(refresh_seconds, canonical_query)
def foo(*, k: tc.dict_of((int, int), [tc.has("^[0-9]+$"), tc.has("^[0-9]+$")])): return functools.reduce(lambda r, t: r and str(t[0][0]) == t[1][0] and str(t[0][1]) == t[1][1], k.items(), True)
def foo(x: tc.dict_of(int, str)) -> tc.dict_of(str, int): return { v: k for k, v in x.items() }
assert foo(k = { (1, 2): ["1", "2"], (3, 4): ["3", "4"]}) assert not foo(k = { (1, 3): ["1", "2"], (3, 4): ["3", "4"]}) assert not foo(k = { (1, 2): ["1", "2"], (3, 4): ["3", "5"]}) with expected(InputParameterError("foo() has got an incompatible value for k: {(1, 2): ['1', '2'], (3, 4): ['3', 'x']}")): foo(k = { (1, 2): ["1", "2"], (3, 4): ["3", "x"]}) with expected(InputParameterError("foo() has got an incompatible value for k: {(1, 2): ['1', '2'], (3,): ['3', '4']}")): foo(k = { (1, 2): ["1", "2"], (3, ): ["3", "4"]}) with expected(InputParameterError("foo() has got an incompatible value for k: {(1, 2): ['1', '2'], (3, 4.0): ['3', '4']}")): foo(k = { (1, 2): ["1", "2"], (3, 4.0): ["3", "4"]}) ################### assert tc.dict_of(int, tc.optional(str))({ 1: "foo", 2: None }) and \ tc.dict_of(int, tc.optional(str)).check({ 1: "foo", 2: None }) assert not tc.dict_of(int, tc.optional(str))({ None: "foo", 2: None }) and \ not tc.dict_of(int, tc.optional(str)).check({ None: "foo", 2: None }) print("ok") ############################################################################ print("enum: ", end="") @typecheck def foo(x: tc.enum(int, 1)) -> tc.enum(1, int): return x
class Packet(dict): _valid_key = by_regex("^[A-Za-z0-9_]+$") _valid_keys = dict_of(_valid_key, str) _bol_b = b"4F36095410830A13" _eol_b = b"92B4782E3B570FD3" @typecheck def __init__(self, **params): assert self._valid_keys(params) self.update(params) ################################### @classmethod @typecheck def load_from_stream(cls, stream: with_attr("readline"), bol_b: bytes, eol_b: bytes): params = {} def decode_line(b): key, value = b.split(b"=", 1) key, value = key.decode("ascii"), a2b_base64(value).decode("utf-8") params[key] = value # the output lines may contain garbage emitted by any java # library and they are therefore filtered and checksummed valid_line = regex(b"^.*(?:" + bol_b + b"|" + cls._bol_b + b")" + b"([0-9A-F]{8})( ?[A-Za-z0-9+/=_]*)" + b"(?:" + eol_b + b"|" + cls._eol_b + b").*$") def get_next_line(prev_crc32 = 0): while True: b = stream.readline() if not b: return None b = b.rstrip() if not b: continue valid_parts = valid_line.findall(b) if len(valid_parts) != 1: pmnc.log.warning("skipping unexpected output: {0:s}".format(str(b)[2:-1])) continue next_crc32, bb = int(valid_parts[0][0], 16), valid_parts[0][1] if next_crc32 != crc32(bb, prev_crc32): pmnc.log.warning("skipping broken output: {0:s}".format(str(b)[2:-1])) continue return bb, next_crc32 curr_lines = [] next_line_crc32 = get_next_line() if next_line_crc32 is None: return None next_line, curr_crc32 = next_line_crc32 while next_line: if next_line.startswith(b" "): if curr_lines: curr_lines.append(next_line[1:]) else: raise Exception("invalid folding") else: if curr_lines: decode_line(b"".join(curr_lines)) del curr_lines[:] curr_lines.append(next_line) next_line_crc32 = get_next_line(curr_crc32) if next_line_crc32 is None: raise Exception("unexpected eof") next_line, curr_crc32 = next_line_crc32 if curr_lines: decode_line(b"".join(curr_lines)) return cls(**params) ################################### @typecheck def save_to_stream(self, stream: with_attr("write", "flush"), fold_width: int): for k, v in self.items(): encoded = k.encode("ascii") + b"=" + b2a_base64(v.encode("utf-8")).rstrip() first_line, encoded = encoded[:fold_width], encoded[fold_width:] stream.write(first_line + b"\n") for folded_line in [ b" " + encoded[i:i+fold_width-1] for i in range(0, len(encoded), fold_width-1) ]: stream.write(folded_line + b"\n") stream.write(b"\n") stream.flush()