def test_rollback_fails():
# rollback hangs
fake_request(1.0)
def execute(res, *args, **kwargs):
1 / 0
def rollback(res, *args, **kwargs):
sleep(2.0)
hooks = hooks_.copy(); hooks["execute"] = execute; hooks["rollback"] = rollback
xa = pmnc.transaction.create()
xa.callable_1(**hooks).execute()
try:
xa.execute()
except ResourceError as e:
assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e))
assert e.participant_index == 0 and e.terminal and not e.recoverable
else:
assert False
assert q.pop(0.0)[:2] == ("connect", 0)
assert q.pop(0.0)[:2] == ("begin_transaction", 1)
assert q.pop(1.0) is None # this is where rollback is called
assert q.pop(2.0)[:2] == ("disconnect", 2)
# rollback throws
fake_request(1.0)
def execute(res, *args, **kwargs):
1 / 0
def rollback(res, *args, **kwargs):
{}["not there"]
hooks = hooks_.copy(); hooks["execute"] = execute; hooks["rollback"] = rollback
xa = pmnc.transaction.create()
xa.callable_1(**hooks).execute()
try:
xa.execute()
except ResourceError as e:
assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e))
assert e.participant_index == 0 and e.terminal and not e.recoverable
else:
assert False
assert q.pop(0.0)[:2] == ("connect", 0)
assert q.pop(0.0)[:2] == ("begin_transaction", 1)
assert q.pop(1.0)[:2] == ("disconnect", 2)
def test_failure():
fake_request(10.0)
def execute(resource, *args, **kwargs):
1 / 0
# failure sequence (the instance is reused): begin_transaction, execute, rollback, disconnect
xa = pmnc.transaction.create(biz = "baz")
xa.callable_1("abc", begin_transaction = begin_transaction, execute = execute,
commit = commit, rollback = rollback, foo = "bar").execute(1, 2, eee = "fff")
try:
xa.execute()
except ResourceError as e:
assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e))
assert not e.recoverable and e.terminal
else:
assert False
# now check the trace
m, c, args, kwargs = q.pop(0.0)
assert m == "begin_transaction" and c == 4
assert args == (xa._xid, )
assert kwargs == dict(transaction_options = { "biz": "baz" }, source_module_name = __name__,
resource_args = ("abc", ), resource_kwargs = { "foo": "bar" })
assert q.pop(1.0) == ("rollback", 5) # rollback is not waited upon, therefore "rollback" may not appear in the queue immediately
assert q.pop(1.0) == ("disconnect", 6)
assert q.pop(1.0) is None
def locate(self, module_name: by_regex("^[A-Za-z0-9_-]{1,128}\\.pyc?$")) -> optional(os_path.isfile):
if module_name in self._listdir(self._cage_directory):
return os_path.join(self._cage_directory, module_name)
elif self._shared_directory and module_name in self._listdir(self._shared_directory):
return os_path.join(self._shared_directory, module_name)
else:
return None # not found
def test_connect_fails():
# connect fails
fake_request(1.0)
xa = pmnc.transaction.create()
xa.callable_2(**hooks_).execute()
try:
xa.execute()
except ResourceError as e:
assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e))
assert e.participant_index == 0 and e.terminal and e.recoverable
else:
assert False
# connect hangs
fake_request(1.0)
xa = pmnc.transaction.create()
xa.callable_3(**hooks_).execute()
try:
xa.execute()
except TransactionExecutionError as e:
assert str(e) == "request deadline waiting for intermediate result from " \
"resource callable_3 in transaction {0:s}".format(xa)
assert e.participant_index == 0
else:
assert False
def current_extents():
return sorted(
[
int(s.split(".")[-1])
for s in listdir(_module_state_dir(__name__))
if by_regex("^.*\\.ext_queue\\.queue\\.[0-9]+$")(s)
]
)
def test_execute_fails():
# execute hangs
fake_request(1.0)
def execute(res, *args, **kwargs):
sleep(2.0)
hooks = hooks_.copy(); hooks["execute"] = execute
xa = pmnc.transaction.create()
xa.callable_1(**hooks).execute()
try:
xa.execute()
except TransactionExecutionError as e:
assert str(e) == "request deadline waiting for intermediate result from " \
"resource callable_1 in transaction {0:s}".format(xa)
assert e.participant_index == 0
else:
assert False
assert q.pop(0.0)[:2] == ("connect", 0)
assert q.pop(0.0)[:2] == ("begin_transaction", 1)
assert q.pop(0.5) is None # this is where execute is called
assert q.pop(1.0)[:2] == ("rollback", 2)
assert q.pop(1.0) is None
# note that the resource instance had not failed and is reused
# execute throws
fake_request(1.0)
def execute(res, *args, **kwargs):
1 / 0
hooks = hooks_.copy(); hooks["execute"] = execute
xa = pmnc.transaction.create()
xa.callable_1(**hooks).execute()
try:
xa.execute()
except ResourceError as e:
assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e))
assert e.participant_index == 0 and e.terminal and not e.recoverable
else:
assert False
assert q.pop(0.0)[:2] == ("begin_transaction", 3)
assert q.pop(1.0)[:2] == ("rollback", 4)
assert q.pop(1.0)[:2] == ("disconnect", 5)
assert q.pop(1.0) is None
def _non_recursive_copy(source_dir, target_dir, mask):
if not os_path.isdir(source_dir):
return
mkdir(target_dir)
matching_file = by_regex("^{0:s}$".format(mask))
for f in listdir(source_dir):
if matching_file(f.lower()):
sf = os_path.join(source_dir, f)
tf = os_path.join(target_dir, f)
if os_path.isfile(sf):
copyfile(sf, tf)
def test_plain_success_failure():
# success
fake_request(1.0)
hooks = hooks_.copy()
xa = pmnc.transaction.create()
xa.callable_1(**hooks).execute()
assert xa.execute() == ("ok", )
assert q.pop(0.0)[:2] == ("connect", 0)
assert q.pop(0.0)[:2] == ("begin_transaction", 1)
assert q.pop(0.0)[:2] == ("execute", 2)
assert q.pop(0.0) == ("commit", 3) # commit is waited upon, therefore "commit" is in the queue
assert q.pop(1.0) is None
# failure
fake_request(1.0)
def execute(res, *args, **kwargs):
1 / 0
hooks = hooks_.copy(); hooks["execute"] = execute
xa = pmnc.transaction.create()
xa.callable_1(**hooks).execute()
try:
xa.execute()
except ResourceError as e:
assert by_regex("^(?:int )?division (?:or modulo )?by zero$")(str(e))
assert e.participant_index == 0 and e.terminal and not e.recoverable # unhandled exception
else:
assert False
assert q.pop(0.0)[:2] == ("begin_transaction", 4)
assert q.pop(1.0) == ("rollback", 5) # rollback is not waited upon, therefore "rollback" may not appear in the queue immediately
assert q.pop(1.0) == ("disconnect", 6)
assert q.pop(1.0) is None
sampler = cls(float(self._period))
self._samplers[key] = sampler
if type(sampler) is not cls: # can't use isinstance b/c RateSampler is a subclass of RawSampler
raise Exception("performance key \"{0:s}\" has already been allocated "
"as {1:s}".format(key, sampler.__class__.__name__))
return sampler
# this method is called whenever the instance has completed collecting
# and returns the summary of the collected statisitics used for display
def dump(self) -> dict:
pass # abstract
###############################################################################
valid_time_key = by_regex("^(interface|resource)\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+_time(\\.failure|\\.success)?$")
valid_rate_key = by_regex("^(interface|resource)\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+_rate(\\.failure|\\.success)?$")
###############################################################################
class CurrentSampler(Sampler):
@typecheck
def tick(self, key: valid_rate_key):
sampler = self._get_sampler(key, RateSampler)
sampler.tick()
@typecheck
def sample(self, key: valid_time_key, value: int):
sampler = self._get_sampler(key, RawSampler)
sampler += value
# complete the response
html.write(_decorate("------------------------------------------------------------------------<br/>\n"))
html.write("</body></html>")
# require a refresh within a configured time
refresh_seconds = pmnc.config.get("refresh_seconds")
response["headers"]["refresh"] = \
"{0:d};URL=/notifications?{1:s}".format(refresh_seconds, canonical_query)
###############################################################################
valid_perf_query_element = "(interface|resource)\\.[A-Za-z0-9_-]+\\.({0:s})=(collapsed|expanded)".\
format("|".join(legends.keys()))
valid_perf_query = by_regex("^({0:s}(&{0:s})*)?$".format(valid_perf_query_element))
valid_ntfy_query = by_regex("^$")
###############################################################################
# this method is called from the HTTP interface for actual request processing
def process_request(request: dict, response: dict):
parsed_url = urlparse(request["url"])
path = parsed_url.path
query = parsed_url.query
html = StringIO()
if path in ("/", "/performance"):
if not valid_perf_query(query):
__all__ = [ "process_request" ]
###############################################################################
import os; from os import path as os_path
if __name__ == "__main__": # add pythomnic/lib to sys.path
import os; import sys
main_module_dir = os.path.dirname(sys.modules["__main__"].__file__) or os.getcwd()
sys.path.insert(0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib")))
import typecheck; from typecheck import by_regex
###############################################################################
valid_url = by_regex("^/(?:(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_-]+\\.html)?$")
###############################################################################
def process_request(request: dict, response: dict):
url = request["url"]
if not valid_url(url):
raise Exception("invalid URL: {0:s}".format(url))
if url == "/":
url = "/index.html"
filename = os_path.join(__cage_dir__, "html", *(url.split("/")[1:]))
import threading; from threading import current_thread
if __name__ == "__main__": # add pythomnic/lib to sys.path
import os; import sys
main_module_dir = os.path.dirname(sys.modules["__main__"].__file__) or os.getcwd()
sys.path.insert(0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib")))
import exc_string; from exc_string import exc_string
import typecheck; from typecheck import typecheck, optional, by_regex, tuple_of
import pmnc.threads; from pmnc.threads import HeavyThread
import pmnc.request; from pmnc.request import fake_request
import pmnc.thread_pool; from pmnc.thread_pool import WorkUnitTimedOut
###############################################################################
valid_cage_name = by_regex("^[A-Za-z0-9_-]{1,32}$")
###############################################################################
class Interface: # reverse RPC interface
@typecheck
def __init__(self, name: str, *,
source_cages: tuple_of(valid_cage_name),
request_timeout: optional(float) = None,
**kwargs):
self._name = name
self._source_cages = source_cages
self._request_timeout = request_timeout or \
r = fake_request(timeout = 1.0, interface = "test")
assert r.interface == "test"
assert not r.expired
sleep(1.1)
assert r.expired
###################################
r = fake_request()
assert r is current_thread()._request
assert isinstance(r, InfiniteRequest)
###################################
r = Request(timeout = 1.0, interface = "test", protocol = "n/a")
assert by_regex("^RQ-[0-9A-F]{4} via test (?:\\+1\\.0|\\+0\\.[1-9])s$")(r.description)
r = Request(timeout = 1.0, interface = "test", protocol = "n/a", description = "some request")
assert by_regex("^RQ-[0-9A-F]{4} \\(some request\\) via test (?:\\+1\\.0|\\+0\\.[1-9])s$")(r.description)
r.describe("some better request")
assert by_regex("^RQ-[0-9A-F]{4} \\(some better request\\) via test (?:\\+1\\.0|\\+0\\.[1-9])s$")(r.description)
sleep(1.5)
assert by_regex("^RQ-[0-9A-F]{4} \\(some better request\\) via test (?:-0\\.[5-9])s$")(r.description)
###################################
print("ok")
################################################################################
def r1(a: by_regex('^[0-9]+$'), _when = "isinstance(a, str)") -> lambda i: i == 99:
return int(a)
import pmnc.timeout
from pmnc.timeout import Timeout
import pmnc.resource_pool
from pmnc.resource_pool import TransactionalResource, ResourceError
import mongodb.connection
from mongodb.connection import *
import mongodb.request
from mongodb.request import *
import mongodb.response
from mongodb.response import *
import mongodb.bson
from mongodb.bson import *
###############################################################################
valid_database = by_regex("^[A-Za-z0-9_]+$")
valid_collection = by_regex("^[A-Za-z0-9_]+(?:\\.[A-Za-z0-9_]+)*$")
empty_string = by_regex("^$")
###############################################################################
class Resource(TransactionalResource): # MongoDB resource
@typecheck
def __init__(self,
name: str,
*,
server_address: (str, int),
connect_timeout: float,
database: valid_database,
username: optional(str) = None,
0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib")))
import exc_string
from exc_string import exc_string
import typecheck
from typecheck import typecheck, optional, by_regex, tuple_of
import pmnc.threads
from pmnc.threads import HeavyThread
import pmnc.request
from pmnc.request import fake_request
import pmnc.thread_pool
from pmnc.thread_pool import WorkUnitTimedOut
###############################################################################
valid_cage_name = by_regex("^[A-Za-z0-9_-]{1,32}$")
###############################################################################
class Interface: # reverse RPC interface
@typecheck
def __init__(self,
name: str,
*,
source_cages: tuple_of(valid_cage_name),
request_timeout: optional(float) = None,
**kwargs):
self._name = name
self._source_cages = source_cages
def current_extents():
return sorted([
int(s.split(".")[-1])
for s in listdir(_module_state_dir(__name__))
if by_regex("^.*\\.ext_queue\\.queue\\.[0-9]+$")(s)
])
assert r.expired
r.remain = 1.0
assert r.remain > 0.5
r.remain = -1.0
assert r.remain == 0.0
###################################
r = fake_request()
assert r is current_thread()._request
assert isinstance(r, InfiniteRequest)
###################################
r = Request(timeout=1.0, interface="test", protocol="n/a")
assert by_regex("^RQ-[0-9A-F]{4} via test (?:\\+1\\.0|\\+0\\.[1-9])s$")(
r.description)
r = Request(timeout=1.0,
interface="test",
protocol="n/a",
description="some request")
assert by_regex(
"^RQ-[0-9A-F]{4} \\(some request\\) via test (?:\\+1\\.0|\\+0\\.[1-9])s$"
)(r.description)
r.describe("some better request")
assert by_regex(
"^RQ-[0-9A-F]{4} \\(some better request\\) via test (?:\\+1\\.0|\\+0\\.[1-9])s$"
)(r.description)
sleep(1.5)
__all__ = [ "execute_sync", "execute_async", "accept", "transmit" ]
###############################################################################
import time; from time import time
if __name__ == "__main__": # add pythomnic/lib to sys.path
import os; import sys
main_module_dir = os.path.dirname(sys.modules["__main__"].__file__) or os.getcwd()
sys.path.insert(0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib")))
import typecheck; from typecheck import optional, by_regex
###############################################################################
valid_cage_name = by_regex("^[A-Za-z0-9_-]{1,32}$")
valid_module_name = by_regex("^[A-Za-z0-9_]{1,64}$")
valid_method_name = by_regex("^[A-Za-z0-9_]{1,64}$")
valid_queue_name = by_regex("^[A-Za-z0-9_-]{1,64}$")
valid_retry_id = by_regex("^RT-[0-9]{14}-[0-9A-F]{12}$")
###############################################################################
# this method is called by the module loader to handle synchronous RPC call
# pmnc("target_cage").module.method(*args, **kwargs)
def execute_sync(target_cage: valid_cage_name, module: valid_module_name,
method: valid_method_name, args: tuple, kwargs: dict):
# execution of synchronous RPC calls to a cage foo is done through
# a separate thread pool rpc__foo, as though each target cage was
# a separate resource; this is beneficial because it gives an
import threading; from threading import Lock
if __name__ == "__main__": # add pythomnic/lib to sys.path
import os; import sys
main_module_dir = os.path.dirname(sys.modules["__main__"].__file__) or os.getcwd()
sys.path.insert(0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib")))
import exc_string; from exc_string import exc_string
import typecheck; from typecheck import by_regex
import interlocked_queue; from interlocked_queue import InterlockedQueue
import pmnc.resource_pool; from pmnc.resource_pool import ResourceError, RPCError
###############################################################################
valid_cage_name = by_regex("^[A-Za-z0-9_-]{1,32}$")
valid_module_name = by_regex("^[A-Za-z0-9_]{1,64}$")
valid_method_name = by_regex("^[A-Za-z0-9_]{1,64}$")
###############################################################################
# module-level state => not reloadable
_rq_queues = {} # one queue per target cage, permanent
_rq_queues_lock = Lock()
_rs_queues = {} # one queue per active request, transient
_rs_queues_lock = Lock()
###############################################################################
class ModuleLoaderError(Exception): pass
class InvalidModuleNameError(ModuleLoaderError): pass
class ModuleNotFoundError(ModuleLoaderError): pass
class ModuleReloadTimedOutError(ModuleLoaderError): pass
class ModuleAccessTimedOutError(ModuleLoaderError): pass
class ModuleFileBrokenError(ModuleLoaderError): pass
class ModuleFileIncompleteError(ModuleLoaderError): pass
class ModuleWithDependenciesError(ModuleLoaderError): pass
class ModuleAlreadyImportedError(ModuleLoaderError): pass
class ModuleNotImportedError(ModuleLoaderError): pass
class InvalidMethodAccessError(ModuleLoaderError): pass
###############################################################################
valid_node_name = by_regex("^[A-Za-z0-9_-]{1,32}$")
valid_cage_name = by_regex("^(?:[A-Za-z0-9_-]{1,32}|\\.shared)$")
valid_cage_name_suffix = by_regex("^(?:[A-Za-z0-9_-]{1,32}(?::retry|:reverse)?|(?::retry|:reverse))$")
valid_module_ext = by_regex("^\\.pyc?$")
###############################################################################
class MethodProxy:
def __init__(self, method, unlock_module, src_module):
self._method, self._unlock_module = method, unlock_module
self._src_module = src_module
def __del__(self):
self.__dict__.pop("_unlock_module", lambda: None)()
class Packet(dict):
_valid_key = by_regex("^[A-Za-z0-9_]+$")
_valid_keys = dict_of(_valid_key, str)
_bol_b = b"4F36095410830A13"
_eol_b = b"92B4782E3B570FD3"
@typecheck
def __init__(self, **params):
assert self._valid_keys(params)
self.update(params)
###################################
@classmethod
@typecheck
def load_from_stream(cls, stream: with_attr("readline"), bol_b: bytes, eol_b: bytes):
params = {}
def decode_line(b):
key, value = b.split(b"=", 1)
key, value = key.decode("ascii"), a2b_base64(value).decode("utf-8")
params[key] = value
# the output lines may contain garbage emitted by any java
# library and they are therefore filtered and checksummed
valid_line = regex(b"^.*(?:" + bol_b + b"|" + cls._bol_b + b")" +
b"([0-9A-F]{8})( ?[A-Za-z0-9+/=_]*)" +
b"(?:" + eol_b + b"|" + cls._eol_b + b").*$")
def get_next_line(prev_crc32 = 0):
while True:
b = stream.readline()
if not b:
return None
b = b.rstrip()
if not b:
continue
valid_parts = valid_line.findall(b)
if len(valid_parts) != 1:
pmnc.log.warning("skipping unexpected output: {0:s}".format(str(b)[2:-1]))
continue
next_crc32, bb = int(valid_parts[0][0], 16), valid_parts[0][1]
if next_crc32 != crc32(bb, prev_crc32):
pmnc.log.warning("skipping broken output: {0:s}".format(str(b)[2:-1]))
continue
return bb, next_crc32
curr_lines = []
next_line_crc32 = get_next_line()
if next_line_crc32 is None:
return None
next_line, curr_crc32 = next_line_crc32
while next_line:
if next_line.startswith(b" "):
if curr_lines:
curr_lines.append(next_line[1:])
else:
raise Exception("invalid folding")
else:
if curr_lines:
decode_line(b"".join(curr_lines))
del curr_lines[:]
curr_lines.append(next_line)
next_line_crc32 = get_next_line(curr_crc32)
if next_line_crc32 is None:
raise Exception("unexpected eof")
next_line, curr_crc32 = next_line_crc32
if curr_lines:
decode_line(b"".join(curr_lines))
return cls(**params)
###################################
@typecheck
def save_to_stream(self, stream: with_attr("write", "flush"), fold_width: int):
for k, v in self.items():
encoded = k.encode("ascii") + b"=" + b2a_base64(v.encode("utf-8")).rstrip()
first_line, encoded = encoded[:fold_width], encoded[fold_width:]
stream.write(first_line + b"\n")
for folded_line in [ b" " + encoded[i:i+fold_width-1]
for i in range(0, len(encoded), fold_width-1) ]:
stream.write(folded_line + b"\n")
stream.write(b"\n")
stream.flush()
