def test_rollback_fails(): # rollback hangs fake_request(1.0) def execute(res, *args, **kwargs): 1 / 0 def rollback(res, *args, **kwargs): sleep(2.0) hooks = hooks_.copy(); hooks["execute"] = execute; hooks["rollback"] = rollback xa = pmnc.transaction.create() xa.callable_1(**hooks).execute() try: xa.execute() except ResourceError as e: assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e)) assert e.participant_index == 0 and e.terminal and not e.recoverable else: assert False assert q.pop(0.0)[:2] == ("connect", 0) assert q.pop(0.0)[:2] == ("begin_transaction", 1) assert q.pop(1.0) is None # this is where rollback is called assert q.pop(2.0)[:2] == ("disconnect", 2) # rollback throws fake_request(1.0) def execute(res, *args, **kwargs): 1 / 0 def rollback(res, *args, **kwargs): {}["not there"] hooks = hooks_.copy(); hooks["execute"] = execute; hooks["rollback"] = rollback xa = pmnc.transaction.create() xa.callable_1(**hooks).execute() try: xa.execute() except ResourceError as e: assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e)) assert e.participant_index == 0 and e.terminal and not e.recoverable else: assert False assert q.pop(0.0)[:2] == ("connect", 0) assert q.pop(0.0)[:2] == ("begin_transaction", 1) assert q.pop(1.0)[:2] == ("disconnect", 2)
def test_failure(): fake_request(10.0) def execute(resource, *args, **kwargs): 1 / 0 # failure sequence (the instance is reused): begin_transaction, execute, rollback, disconnect xa = pmnc.transaction.create(biz = "baz") xa.callable_1("abc", begin_transaction = begin_transaction, execute = execute, commit = commit, rollback = rollback, foo = "bar").execute(1, 2, eee = "fff") try: xa.execute() except ResourceError as e: assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e)) assert not e.recoverable and e.terminal else: assert False # now check the trace m, c, args, kwargs = q.pop(0.0) assert m == "begin_transaction" and c == 4 assert args == (xa._xid, ) assert kwargs == dict(transaction_options = { "biz": "baz" }, source_module_name = __name__, resource_args = ("abc", ), resource_kwargs = { "foo": "bar" }) assert q.pop(1.0) == ("rollback", 5) # rollback is not waited upon, therefore "rollback" may not appear in the queue immediately assert q.pop(1.0) == ("disconnect", 6) assert q.pop(1.0) is None
def locate(self, module_name: by_regex("^[A-Za-z0-9_-]{1,128}\\.pyc?$")) -> optional(os_path.isfile): if module_name in self._listdir(self._cage_directory): return os_path.join(self._cage_directory, module_name) elif self._shared_directory and module_name in self._listdir(self._shared_directory): return os_path.join(self._shared_directory, module_name) else: return None # not found
def test_connect_fails(): # connect fails fake_request(1.0) xa = pmnc.transaction.create() xa.callable_2(**hooks_).execute() try: xa.execute() except ResourceError as e: assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e)) assert e.participant_index == 0 and e.terminal and e.recoverable else: assert False # connect hangs fake_request(1.0) xa = pmnc.transaction.create() xa.callable_3(**hooks_).execute() try: xa.execute() except TransactionExecutionError as e: assert str(e) == "request deadline waiting for intermediate result from " \ "resource callable_3 in transaction {0:s}".format(xa) assert e.participant_index == 0 else: assert False
def current_extents(): return sorted( [ int(s.split(".")[-1]) for s in listdir(_module_state_dir(__name__)) if by_regex("^.*\\.ext_queue\\.queue\\.[0-9]+$")(s) ] )
def test_execute_fails(): # execute hangs fake_request(1.0) def execute(res, *args, **kwargs): sleep(2.0) hooks = hooks_.copy(); hooks["execute"] = execute xa = pmnc.transaction.create() xa.callable_1(**hooks).execute() try: xa.execute() except TransactionExecutionError as e: assert str(e) == "request deadline waiting for intermediate result from " \ "resource callable_1 in transaction {0:s}".format(xa) assert e.participant_index == 0 else: assert False assert q.pop(0.0)[:2] == ("connect", 0) assert q.pop(0.0)[:2] == ("begin_transaction", 1) assert q.pop(0.5) is None # this is where execute is called assert q.pop(1.0)[:2] == ("rollback", 2) assert q.pop(1.0) is None # note that the resource instance had not failed and is reused # execute throws fake_request(1.0) def execute(res, *args, **kwargs): 1 / 0 hooks = hooks_.copy(); hooks["execute"] = execute xa = pmnc.transaction.create() xa.callable_1(**hooks).execute() try: xa.execute() except ResourceError as e: assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e)) assert e.participant_index == 0 and e.terminal and not e.recoverable else: assert False assert q.pop(0.0)[:2] == ("begin_transaction", 3) assert q.pop(1.0)[:2] == ("rollback", 4) assert q.pop(1.0)[:2] == ("disconnect", 5) assert q.pop(1.0) is None
def _non_recursive_copy(source_dir, target_dir, mask): if not os_path.isdir(source_dir): return mkdir(target_dir) matching_file = by_regex("^{0:s}$".format(mask)) for f in listdir(source_dir): if matching_file(f.lower()): sf = os_path.join(source_dir, f) tf = os_path.join(target_dir, f) if os_path.isfile(sf): copyfile(sf, tf)
def test_plain_success_failure(): # success fake_request(1.0) hooks = hooks_.copy() xa = pmnc.transaction.create() xa.callable_1(**hooks).execute() assert xa.execute() == ("ok", ) assert q.pop(0.0)[:2] == ("connect", 0) assert q.pop(0.0)[:2] == ("begin_transaction", 1) assert q.pop(0.0)[:2] == ("execute", 2) assert q.pop(0.0) == ("commit", 3) # commit is waited upon, therefore "commit" is in the queue assert q.pop(1.0) is None # failure fake_request(1.0) def execute(res, *args, **kwargs): 1 / 0 hooks = hooks_.copy(); hooks["execute"] = execute xa = pmnc.transaction.create() xa.callable_1(**hooks).execute() try: xa.execute() except ResourceError as e: assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e)) assert e.participant_index == 0 and e.terminal and not e.recoverable # unhandled exception else: assert False assert q.pop(0.0)[:2] == ("begin_transaction", 4) assert q.pop(1.0) == ("rollback", 5) # rollback is not waited upon, therefore "rollback" may not appear in the queue immediately assert q.pop(1.0) == ("disconnect", 6) assert q.pop(1.0) is None
sampler = cls(float(self._period)) self._samplers[key] = sampler if type(sampler) is not cls: # can't use isinstance b/c RateSampler is a subclass of RawSampler raise Exception("performance key \"{0:s}\" has already been allocated " "as {1:s}".format(key, sampler.__class__.__name__)) return sampler # this method is called whenever the instance has completed collecting # and returns the summary of the collected statisitics used for display def dump(self) -> dict: pass # abstract ############################################################################### valid_time_key = by_regex("^(interface|resource)\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+_time(\\.failure|\\.success)?$") valid_rate_key = by_regex("^(interface|resource)\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+_rate(\\.failure|\\.success)?$") ############################################################################### class CurrentSampler(Sampler): @typecheck def tick(self, key: valid_rate_key): sampler = self._get_sampler(key, RateSampler) sampler.tick() @typecheck def sample(self, key: valid_time_key, value: int): sampler = self._get_sampler(key, RawSampler) sampler += value
# complete the response html.write(_decorate("------------------------------------------------------------------------<br/>\n")) html.write("</body></html>") # require a refresh within a configured time refresh_seconds = pmnc.config.get("refresh_seconds") response["headers"]["refresh"] = \ "{0:d};URL=/notifications?{1:s}".format(refresh_seconds, canonical_query) ############################################################################### valid_perf_query_element = "(interface|resource)\\.[A-Za-z0-9_-]+\\.({0:s})=(collapsed|expanded)".\ format("|".join(legends.keys())) valid_perf_query = by_regex("^({0:s}(&{0:s})*)?$".format(valid_perf_query_element)) valid_ntfy_query = by_regex("^$") ############################################################################### # this method is called from the HTTP interface for actual request processing def process_request(request: dict, response: dict): parsed_url = urlparse(request["url"]) path = parsed_url.path query = parsed_url.query html = StringIO() if path in ("/", "/performance"): if not valid_perf_query(query):
__all__ = [ "process_request" ] ############################################################################### import os; from os import path as os_path if __name__ == "__main__": # add pythomnic/lib to sys.path import os; import sys main_module_dir = os.path.dirname(sys.modules["__main__"].__file__) or os.getcwd() sys.path.insert(0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib"))) import typecheck; from typecheck import by_regex ############################################################################### valid_url = by_regex("^/(?:(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_-]+\\.html)?$") ############################################################################### def process_request(request: dict, response: dict): url = request["url"] if not valid_url(url): raise Exception("invalid URL: {0:s}".format(url)) if url == "/": url = "/index.html" filename = os_path.join(__cage_dir__, "html", *(url.split("/")[1:]))
import threading; from threading import current_thread if __name__ == "__main__": # add pythomnic/lib to sys.path import os; import sys main_module_dir = os.path.dirname(sys.modules["__main__"].__file__) or os.getcwd() sys.path.insert(0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib"))) import exc_string; from exc_string import exc_string import typecheck; from typecheck import typecheck, optional, by_regex, tuple_of import pmnc.threads; from pmnc.threads import HeavyThread import pmnc.request; from pmnc.request import fake_request import pmnc.thread_pool; from pmnc.thread_pool import WorkUnitTimedOut ############################################################################### valid_cage_name = by_regex("^[A-Za-z0-9_-]{1,32}$") ############################################################################### class Interface: # reverse RPC interface @typecheck def __init__(self, name: str, *, source_cages: tuple_of(valid_cage_name), request_timeout: optional(float) = None, **kwargs): self._name = name self._source_cages = source_cages self._request_timeout = request_timeout or \
r = fake_request(timeout = 1.0, interface = "test") assert r.interface == "test" assert not r.expired sleep(1.1) assert r.expired ################################### r = fake_request() assert r is current_thread()._request assert isinstance(r, InfiniteRequest) ################################### r = Request(timeout = 1.0, interface = "test", protocol = "n/a") assert by_regex("^RQ-[0-9A-F]{4} via test (?:\\+1\\.0|\\+0\\.[1-9])s$")(r.description) r = Request(timeout = 1.0, interface = "test", protocol = "n/a", description = "some request") assert by_regex("^RQ-[0-9A-F]{4} \\(some request\\) via test (?:\\+1\\.0|\\+0\\.[1-9])s$")(r.description) r.describe("some better request") assert by_regex("^RQ-[0-9A-F]{4} \\(some better request\\) via test (?:\\+1\\.0|\\+0\\.[1-9])s$")(r.description) sleep(1.5) assert by_regex("^RQ-[0-9A-F]{4} \\(some better request\\) via test (?:-0\\.[5-9])s$")(r.description) ################################### print("ok") ################################################################################
def r1(a: by_regex('^[0-9]+$'), _when = "isinstance(a, str)") -> lambda i: i == 99: return int(a)
import pmnc.timeout from pmnc.timeout import Timeout import pmnc.resource_pool from pmnc.resource_pool import TransactionalResource, ResourceError import mongodb.connection from mongodb.connection import * import mongodb.request from mongodb.request import * import mongodb.response from mongodb.response import * import mongodb.bson from mongodb.bson import * ############################################################################### valid_database = by_regex("^[A-Za-z0-9_]+$") valid_collection = by_regex("^[A-Za-z0-9_]+(?:\\.[A-Za-z0-9_]+)*$") empty_string = by_regex("^$") ############################################################################### class Resource(TransactionalResource): # MongoDB resource @typecheck def __init__(self, name: str, *, server_address: (str, int), connect_timeout: float, database: valid_database, username: optional(str) = None,
0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib"))) import exc_string from exc_string import exc_string import typecheck from typecheck import typecheck, optional, by_regex, tuple_of import pmnc.threads from pmnc.threads import HeavyThread import pmnc.request from pmnc.request import fake_request import pmnc.thread_pool from pmnc.thread_pool import WorkUnitTimedOut ############################################################################### valid_cage_name = by_regex("^[A-Za-z0-9_-]{1,32}$") ############################################################################### class Interface: # reverse RPC interface @typecheck def __init__(self, name: str, *, source_cages: tuple_of(valid_cage_name), request_timeout: optional(float) = None, **kwargs): self._name = name self._source_cages = source_cages
def current_extents(): return sorted([ int(s.split(".")[-1]) for s in listdir(_module_state_dir(__name__)) if by_regex("^.*\\.ext_queue\\.queue\\.[0-9]+$")(s) ])
assert r.expired r.remain = 1.0 assert r.remain > 0.5 r.remain = -1.0 assert r.remain == 0.0 ################################### r = fake_request() assert r is current_thread()._request assert isinstance(r, InfiniteRequest) ################################### r = Request(timeout=1.0, interface="test", protocol="n/a") assert by_regex("^RQ-[0-9A-F]{4} via test (?:\\+1\\.0|\\+0\\.[1-9])s$")( r.description) r = Request(timeout=1.0, interface="test", protocol="n/a", description="some request") assert by_regex( "^RQ-[0-9A-F]{4} \\(some request\\) via test (?:\\+1\\.0|\\+0\\.[1-9])s$" )(r.description) r.describe("some better request") assert by_regex( "^RQ-[0-9A-F]{4} \\(some better request\\) via test (?:\\+1\\.0|\\+0\\.[1-9])s$" )(r.description) sleep(1.5)
__all__ = [ "execute_sync", "execute_async", "accept", "transmit" ] ############################################################################### import time; from time import time if __name__ == "__main__": # add pythomnic/lib to sys.path import os; import sys main_module_dir = os.path.dirname(sys.modules["__main__"].__file__) or os.getcwd() sys.path.insert(0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib"))) import typecheck; from typecheck import optional, by_regex ############################################################################### valid_cage_name = by_regex("^[A-Za-z0-9_-]{1,32}$") valid_module_name = by_regex("^[A-Za-z0-9_]{1,64}$") valid_method_name = by_regex("^[A-Za-z0-9_]{1,64}$") valid_queue_name = by_regex("^[A-Za-z0-9_-]{1,64}$") valid_retry_id = by_regex("^RT-[0-9]{14}-[0-9A-F]{12}$") ############################################################################### # this method is called by the module loader to handle synchronous RPC call # pmnc("target_cage").module.method(*args, **kwargs) def execute_sync(target_cage: valid_cage_name, module: valid_module_name, method: valid_method_name, args: tuple, kwargs: dict): # execution of synchronous RPC calls to a cage foo is done through # a separate thread pool rpc__foo, as though each target cage was # a separate resource; this is beneficial because it gives an
import threading; from threading import Lock if __name__ == "__main__": # add pythomnic/lib to sys.path import os; import sys main_module_dir = os.path.dirname(sys.modules["__main__"].__file__) or os.getcwd() sys.path.insert(0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib"))) import exc_string; from exc_string import exc_string import typecheck; from typecheck import by_regex import interlocked_queue; from interlocked_queue import InterlockedQueue import pmnc.resource_pool; from pmnc.resource_pool import ResourceError, RPCError ############################################################################### valid_cage_name = by_regex("^[A-Za-z0-9_-]{1,32}$") valid_module_name = by_regex("^[A-Za-z0-9_]{1,64}$") valid_method_name = by_regex("^[A-Za-z0-9_]{1,64}$") ############################################################################### # module-level state => not reloadable _rq_queues = {} # one queue per target cage, permanent _rq_queues_lock = Lock() _rs_queues = {} # one queue per active request, transient _rs_queues_lock = Lock() ###############################################################################
class ModuleLoaderError(Exception): pass class InvalidModuleNameError(ModuleLoaderError): pass class ModuleNotFoundError(ModuleLoaderError): pass class ModuleReloadTimedOutError(ModuleLoaderError): pass class ModuleAccessTimedOutError(ModuleLoaderError): pass class ModuleFileBrokenError(ModuleLoaderError): pass class ModuleFileIncompleteError(ModuleLoaderError): pass class ModuleWithDependenciesError(ModuleLoaderError): pass class ModuleAlreadyImportedError(ModuleLoaderError): pass class ModuleNotImportedError(ModuleLoaderError): pass class InvalidMethodAccessError(ModuleLoaderError): pass ############################################################################### valid_node_name = by_regex("^[A-Za-z0-9_-]{1,32}$") valid_cage_name = by_regex("^(?:[A-Za-z0-9_-]{1,32}|\\.shared)$") valid_cage_name_suffix = by_regex("^(?:[A-Za-z0-9_-]{1,32}(?::retry|:reverse)?|(?::retry|:reverse))$") valid_module_ext = by_regex("^\\.pyc?$") ############################################################################### class MethodProxy: def __init__(self, method, unlock_module, src_module): self._method, self._unlock_module = method, unlock_module self._src_module = src_module def __del__(self): self.__dict__.pop("_unlock_module", lambda: None)()
class Packet(dict): _valid_key = by_regex("^[A-Za-z0-9_]+$") _valid_keys = dict_of(_valid_key, str) _bol_b = b"4F36095410830A13" _eol_b = b"92B4782E3B570FD3" @typecheck def __init__(self, **params): assert self._valid_keys(params) self.update(params) ################################### @classmethod @typecheck def load_from_stream(cls, stream: with_attr("readline"), bol_b: bytes, eol_b: bytes): params = {} def decode_line(b): key, value = b.split(b"=", 1) key, value = key.decode("ascii"), a2b_base64(value).decode("utf-8") params[key] = value # the output lines may contain garbage emitted by any java # library and they are therefore filtered and checksummed valid_line = regex(b"^.*(?:" + bol_b + b"|" + cls._bol_b + b")" + b"([0-9A-F]{8})( ?[A-Za-z0-9+/=_]*)" + b"(?:" + eol_b + b"|" + cls._eol_b + b").*$") def get_next_line(prev_crc32 = 0): while True: b = stream.readline() if not b: return None b = b.rstrip() if not b: continue valid_parts = valid_line.findall(b) if len(valid_parts) != 1: pmnc.log.warning("skipping unexpected output: {0:s}".format(str(b)[2:-1])) continue next_crc32, bb = int(valid_parts[0][0], 16), valid_parts[0][1] if next_crc32 != crc32(bb, prev_crc32): pmnc.log.warning("skipping broken output: {0:s}".format(str(b)[2:-1])) continue return bb, next_crc32 curr_lines = [] next_line_crc32 = get_next_line() if next_line_crc32 is None: return None next_line, curr_crc32 = next_line_crc32 while next_line: if next_line.startswith(b" "): if curr_lines: curr_lines.append(next_line[1:]) else: raise Exception("invalid folding") else: if curr_lines: decode_line(b"".join(curr_lines)) del curr_lines[:] curr_lines.append(next_line) next_line_crc32 = get_next_line(curr_crc32) if next_line_crc32 is None: raise Exception("unexpected eof") next_line, curr_crc32 = next_line_crc32 if curr_lines: decode_line(b"".join(curr_lines)) return cls(**params) ################################### @typecheck def save_to_stream(self, stream: with_attr("write", "flush"), fold_width: int): for k, v in self.items(): encoded = k.encode("ascii") + b"=" + b2a_base64(v.encode("utf-8")).rstrip() first_line, encoded = encoded[:fold_width], encoded[fold_width:] stream.write(first_line + b"\n") for folded_line in [ b" " + encoded[i:i+fold_width-1] for i in range(0, len(encoded), fold_width-1) ]: stream.write(folded_line + b"\n") stream.write(b"\n") stream.flush()