def test_execute_success():
fake_request(10.0)
request_id = None
response = None
def caller(*args, **kwargs):
fake_request(6.0)
nonlocal request_id, response
request_id = pmnc.request.unique_id
pmnc.request.parameters["AAA"] = "BBB"
pmnc.request.describe("my request")
response = pmnc.__getattr__(__name__).execute_reverse("good_cage", "module", "method", args, kwargs)
assert "good_cage" not in _rq_queues
th = HeavyThread(target = caller, args = (1, "foo"), kwargs = { "biz": "baz" })
th.start()
try:
sleep(2.0)
assert "good_cage" in _rq_queues
assert request_id in _rs_queues
req_id, req = _rq_queues["good_cage"].pop()
assert req_id == request_id
assert abs(req["request"].pop("deadline") - time() - 4.0) < 1.0
assert req == dict \
(
source_cage = __cage__,
target_cage = "good_cage",
module = "module",
method = "method",
args = (1, "foo"),
kwargs = { "biz": "baz" },
request = dict(protocol = pmnc.request.protocol,
interface = pmnc.request.interface,
unique_id = request_id,
description = "my request",
parameters = dict(auth_tokens = {}, AAA = "BBB")),
)
_rs_queues[request_id].push({ "result": "RESULT" })
sleep(2.0)
assert "good_cage" in _rq_queues
assert request_id not in _rs_queues
finally:
th.stop()
assert response == "RESULT"
assert "good_cage" in _rq_queues
assert request_id not in _rs_queues
def start():
global _state_thread
if not _no_bssdb3:
_state_thread = HeavyThread(target=_state_thread_proc, name="state")
_state_thread.start()
else:
pmnc.log.message("module bsddb3 could not be imported, " "persistent state is disabled")
def test_WorkSources_end_work():
current_thread().stopped = lambda: False
wss = WorkSources(2, 30.0)
wss.add_work(1)
assert wss.begin_work(0.0) == (False, 1)
wss.add_work(0)
assert wss.begin_work(0.0) == (False, 0)
assert wss.begin_work(0.0) == (False, None)
wss.add_work(1)
assert wss.begin_work(0.0) == (False, None)
th_started = Event()
th_got_work = Event()
def th_proc():
th_started.set()
assert wss.begin_work(10.0) == (False, 1)
th_got_work.set()
th = HeavyThread(target = th_proc)
th.start()
th_started.wait()
t = Timeout(30.0)
wss.end_work(1)
th_got_work.wait()
assert t.remain > 29.0, t.remain
th.stop()
def start(self): # a separate thread is started to poll each source cage
self._pollers = []
for source_cage in self._source_cages:
poller = HeavyThread(target = self._poller_proc, args = (source_cage, ),
name = "{0:s}:{1:s}".format(self._name, source_cage))
self._pollers.append(poller)
for poller in self._pollers:
poller.start()
def start():
global _state_thread
if not _no_bssdb3:
with _state_thread_lock:
if not _state_thread:
_state_thread = HeavyThread(target=_state_thread_proc,
name="state")
_state_thread.start()
else:
pmnc.log.message("module bsddb3 could not be imported, "
"persistent state is disabled")
def start(self): # a separate thread is started to poll each source cage
self._pollers = []
for source_cage in self._source_cages:
poller = HeavyThread(target=self._poller_proc,
args=(source_cage, ),
name="{0:s}:{1:s}".format(
self._name, source_cage))
self._pollers.append(poller)
for poller in self._pollers:
poller.start()
def test_WorkSources_stop_thread():
th_started = Event()
def th_proc():
wss = WorkSources(1, 30.0)
th_started.set()
assert wss.begin_work(10.0) == (True, None)
th = HeavyThread(target = th_proc)
th.start()
t = Timeout(30.0)
th_started.wait()
sleep(1.0)
th.stop()
assert t.remain > 25.0
def test_WorkSources_signal_kick():
th_started = Event()
th_got_work = Event()
wss = WorkSources(1, 30.0)
def th_proc():
th_started.set()
assert wss.begin_work(10.0) == (False, 0)
th_got_work.set()
th = HeavyThread(target = th_proc)
th.start()
th_started.wait()
t = Timeout(30.0)
wss.add_work(0)
th_got_work.wait()
assert t.remain > 29.0
th.stop()
def test_WorkSources_end_work():
current_thread().stopped = lambda: False
wss = WorkSources(2, 30.0)
wss.add_work(1)
assert wss.begin_work(0.0) == (False, 1)
wss.add_work(0)
assert wss.begin_work(0.0) == (False, 0)
assert wss.begin_work(0.0) == (False, None)
wss.add_work(1)
assert wss.begin_work(0.0) == (False, None)
th_started = Event()
th_got_work = Event()
def th_proc():
th_started.set()
assert wss.begin_work(10.0) == (False, 1)
th_got_work.set()
th = HeavyThread(target=th_proc)
th.start()
th_started.wait()
t = Timeout(30.0)
wss.end_work(1)
th_got_work.wait()
assert t.remain > 29.0, t.remain
th.stop()
def test_cross_db_deadlock():
fake_request(30.0)
db1 = pmnc.state.get_database("db1")
db2 = pmnc.state.get_queue("db2", re_len=6144)
def f(txn):
db1.put(b"key", b"value_1", txn)
Timeout(3.0).wait()
db2.append(pickle("item_1"), txn)
def g(txn):
db2.append(pickle("item_2"), txn)
Timeout(3.0).wait()
db1.put(b"key", b"value_2", txn)
th_f = HeavyThread(target=lambda: pmnc.state.implicit_transaction(f))
th_g = HeavyThread(target=lambda: pmnc.state.implicit_transaction(g))
th_f.start()
th_g.start()
th_f.join()
th_g.join()
# now see what went through
def fetch_result(txn):
value = db1.get(b"key", None, txn)
item1 = _pop(txn, db2, None, unpickle)
item2 = _pop(txn, db2, None, unpickle)
return value, item1, item2
value, item1, item2 = pmnc.state.implicit_transaction(fetch_result)
assert value in (b"value_1", b"value_2")
assert (item1, item2) in (("item_1", "item_2"), ("item_2", "item_1"))
def test_WorkSources_stop_thread():
th_started = Event()
def th_proc():
wss = WorkSources(1, 30.0)
th_started.set()
assert wss.begin_work(10.0) == (True, None)
th = HeavyThread(target=th_proc)
th.start()
t = Timeout(30.0)
th_started.wait()
sleep(1.0)
th.stop()
assert t.remain > 25.0
def test_WorkSources_signal_kick():
th_started = Event()
th_got_work = Event()
wss = WorkSources(1, 30.0)
def th_proc():
th_started.set()
assert wss.begin_work(10.0) == (False, 0)
th_got_work.set()
th = HeavyThread(target=th_proc)
th.start()
th_started.wait()
t = Timeout(30.0)
wss.add_work(0)
th_got_work.wait()
assert t.remain > 29.0
th.stop()
def start(self):
self._probe_thread = HeavyThread(
target=self._probe_thread_proc, name="health_monitor:probe"
) # always called "health_monitor"
self._probe_thread.start()
class HealthMonitor:
def __init__(self, **kwargs):
self._rpc_interface = pmnc.interfaces.get_interface("rpc")
if self._rpc_interface is None:
raise Exception("health monitor requires enabled rpc interface")
self._probe_thread_pool = pmnc.shared_pools.get_private_thread_pool()
self._up_cages = {} # { cage: { node: { location: ..., probe_result: ... } } }
self._up_down_queue = InterlockedQueue()
self._request_timeout = pmnc.config_interfaces.get("request_timeout") # this is now static
if pmnc.request.self_test == __name__: # self-test
self._process_event = kwargs["process_event"]
self._probe_cage = kwargs["probe_cage"]
###################################
def start(self):
self._probe_thread = HeavyThread(
target=self._probe_thread_proc, name="health_monitor:probe"
) # always called "health_monitor"
self._probe_thread.start()
def stop(self):
self._probe_thread.stop()
###################################
# this method is executed in a private thread and is scheduling probe calls
# to cages known to the RPC interface or previously probed and found to be up
def _probe_thread_proc(self):
per_cage_interval = 0.0
# calls to _poll_up_down_queue are interleaved and allow this thread
# to maintain structures such as _up_cages in response to events
# posted by the probe threads to the _up_down_queue
while self._poll_up_down_queue(per_cage_interval):
try:
# extract all cages currently known to the rpc interface and
# merge them with cages previously probed and found to be up,
# except for the health_monitor cage itself should be skipped
probe_cages = {
known_cage: {
known_node: dict(location=known_location, probe_result=None)
for known_node, known_location in self._rpc_interface.get_nodes(known_cage).items()
}
for known_cage in self._rpc_interface.get_cages()
if known_cage != "health_monitor"
}
self._merge_cages(probe_cages, self._up_cages)
probe_period = pmnc.config.get("probe_period")
per_cage_interval = probe_period / (len(probe_cages) + 1)
# walk through all cages to be probed and schedule calls to probe
# to a private thread pool using fake unregistered requests
for cage, nodes in probe_cages.items():
for node, cage_info in nodes.items():
cage_location = cage_info["location"]
# note that the requests created here are not registered with
# interfaces and enqueued to a different pool too, they are
# therefore entitled to termination without warning at shutdown,
# this is ok, because they do no useful work for the clients
request = Request(
timeout=self._request_timeout,
interface="__health_monitor__",
protocol="n/a",
parameters=dict(auth_tokens=dict()),
description="probing cage {0:s} at {1:s}".format(cage, cage_location),
)
self._probe_thread_pool.enqueue(
request, self.wu_probe_cage, (node, cage, cage_location, cage_info["probe_result"]), {}
)
# then again yield to polling the queue for a while
if not self._poll_up_down_queue(per_cage_interval):
break
except:
pmnc.log.error(exc_string()) # log and ignore
###################################
# this method merges cages known to the RPC interface with cages
# previously probed and known to be up, such merging is necessary
# because if a cage dies just before its next advertisement broadcast,
# it would disappear from known, but will not be probed again and
# hence thought to be up forever
@staticmethod
def _merge_cages(known_cages: dict, up_cages: dict):
probe_cages = known_cages # merging in place
for up_cage, up_nodes in up_cages.items():
for up_node, up_cage_info in up_nodes.items():
probe_nodes = probe_cages.setdefault(up_cage, {})
if up_node in probe_nodes:
cage_info = probe_nodes[up_node]
if cage_info["location"] == up_cage_info["location"]:
cage_info.update(probe_result=up_cage_info["probe_result"])
else:
cage_info.update(probe_result="restarted") # note this case
else:
probe_nodes[up_node] = up_cage_info
###################################
# a call to this method is enqueued to a private thread pool
# for each cage to probe on every pass of _probe_thread
def wu_probe_cage(self, node, cage, location, prev_probe_result):
if pmnc.request.expired: # no need to report anything for a probing request
return
pmnc.log.debug("sending probe")
try:
probe_result = self._probe_cage(node, cage, location)
except:
pmnc.log.warning("probe failed: {0:s}".format(exc_string()))
self._up_down_queue.push((node, cage, "down"))
else:
pmnc.log.debug("probe returned successfully")
if prev_probe_result == "restarted": # if the cage has restarted
self._up_down_queue.push((node, cage, "down")) # we push "down" event first
self._up_down_queue.push((node, cage, "up", location, probe_result))
###################################
# this method is invoked by one of the private pool threads
# to send the actual probe call to the cage being probed
@typecheck
def _probe_cage(self, node, cage, location) -> dict:
# health monitor has to create rpc resources manually, not using
# pmnc(cage) syntax, because we need to access exact cage at exact
# node and location (i.e. host and port) and to avoid discovery
connect_timeout = pmnc.config_resource_rpc.get("discovery_timeout")
rpc = pmnc.protocol_rpc.Resource(
"{0:s}.{1:s}".format(node, cage),
broadcast_address=("n/a", 0),
discovery_timeout=connect_timeout,
multiple_timeout_allowance=0.0,
flock_id="unused",
exact_locations={cage: location}, # this prevents discovery
pool__resource_name=cage,
)
rpc.connect()
try:
rpc.begin_transaction(
"", source_module_name=__name__, transaction_options={}, resource_args=(), resource_kwargs={}
)
try:
probe_result = rpc.health_monitor_event.probe() # there, an RPC call
except:
rpc.rollback()
raise
else:
rpc.commit()
finally:
rpc.disconnect()
return probe_result # if the cage returns anything but a dict, it is considered a failure
###################################
# this method is called by the _probe_thread during its idle times
# to fetch up/down events posted to the _up_down_queue by the probe
# threads and in response to maintain structures such as _up_cages
def _poll_up_down_queue(self, timeout: float) -> bool: # returns "should keep running"
poll_timeout = Timeout(timeout)
while not poll_timeout.expired:
pop_timeout = Timeout(min(poll_timeout.remain, 1.0))
while not pop_timeout.expired:
event = pop_timeout.pop(self._up_down_queue)
if event is not None:
try:
node, cage, up_down, *args = event
if up_down == "up":
location, probe_result = args
# add the cage to cages known to be up and schedule
# application notification call if it was down or
# returned a different probe result
cage_info = self._up_cages.setdefault(cage, {}).setdefault(node, {})
if not cage_info or cage_info["probe_result"] != probe_result:
self._schedule_up_down_event(node, cage, "up", probe_result)
cage_info.update(location=location, probe_result=probe_result)
elif up_down == "down":
# remove the cage from cages known to be up and schedule
# application notification call it was up
if self._up_cages.setdefault(cage, {}).pop(node, None):
self._schedule_up_down_event(node, cage, "down")
except:
pmnc.log.error(exc_string()) # log and ignore
if current_thread().stopped():
return False
return True
###################################
# this method is called by the _probe_thread in response to change
# of some cage's state detected in _poll_up_down_queue
def _schedule_up_down_event(self, node, cage, up_down, probe_result=None):
# application notification invokes methods from health_monitor_event module
# and must be executed just like a regular request from some interface
request = pmnc.interfaces.begin_request(
timeout=self._request_timeout,
interface="__health_monitor__",
protocol="n/a",
parameters=dict(auth_tokens=dict()),
description="cage {0:s}.{1:s} is {2:s}".format(node, cage, up_down),
)
# note that this request is not waited upon
pmnc.interfaces.enqueue(request, self.wu_process_event, (node, cage, up_down, probe_result))
###################################
# this method is invoked by one of the interfaces pool threads to register
# the event of some cage going up or down by calling an appropriate method
# from the health_monitor_event module
@typecheck
def wu_process_event(self, node: str, cage: str, up_down: one_of("up", "down"), probe_result: optional(dict)):
try:
# see for how long the request was on the execution queue up to this moment
# and whether it has expired in the meantime, if it did there is no reason
# to proceed and we simply bail out
if pmnc.request.expired:
pmnc.log.error("request has expired and will not be processed")
success = False
return # goes through finally section below
with pmnc.performance.request_processing():
self._process_event(node, cage, up_down, probe_result)
except:
pmnc.log.error(exc_string()) # log and ignore
success = False
else:
success = True
finally: # the request ends itself
pmnc.interfaces.end_request(success) # possibly way after deadline
###################################
def _process_event(self, node, cage, up_down, probe_result):
if up_down == "up":
pmnc.health_monitor_event.cage_up(node, cage, probe_result)
elif up_down == "down":
pmnc.health_monitor_event.cage_down(node, cage)
def start(self):
self._probe_thread = HeavyThread(target = self._probe_thread_proc,
name = "health_monitor:probe") # always called "health_monitor"
self._probe_thread.start()
class HealthMonitor:
def __init__(self, **kwargs):
self._rpc_interface = pmnc.interfaces.get_interface("rpc")
if self._rpc_interface is None:
raise Exception("health monitor requires enabled rpc interface")
self._probe_thread_pool = pmnc.shared_pools.get_private_thread_pool()
self._up_cages = {} # { cage: { node: { location: ..., probe_result: ... } } }
self._up_down_queue = InterlockedQueue()
self._request_timeout = pmnc.config_interfaces.get("request_timeout") # this is now static
if pmnc.request.self_test == __name__: # self-test
self._process_event = kwargs["process_event"]
self._probe_cage = kwargs["probe_cage"]
###################################
def start(self):
self._probe_thread = HeavyThread(target = self._probe_thread_proc,
name = "health_monitor:probe") # always called "health_monitor"
self._probe_thread.start()
def stop(self):
self._probe_thread.stop()
###################################
# this method is executed in a private thread and is scheduling probe calls
# to cages known to the RPC interface or previously probed and found to be up
def _probe_thread_proc(self):
per_cage_interval = 0.0
# calls to _poll_up_down_queue are interleaved and allow this thread
# to maintain structures such as _up_cages in response to events
# posted by the probe threads to the _up_down_queue
while self._poll_up_down_queue(per_cage_interval):
try:
# extract all cages currently known to the rpc interface and
# merge them with cages previously probed and found to be up,
# except for the health_monitor cage itself should be skipped
probe_cages = \
{ known_cage: { known_node: dict(location = known_location, probe_result = None)
for known_node, known_location in self._rpc_interface.get_nodes(known_cage).items() }
for known_cage in self._rpc_interface.get_cages() if known_cage != "health_monitor" }
self._merge_cages(probe_cages, self._up_cages)
probe_period = pmnc.config.get("probe_period")
per_cage_interval = probe_period / (len(probe_cages) + 1)
# walk through all cages to be probed and schedule calls to probe
# to a private thread pool using fake unregistered requests
for cage, nodes in probe_cages.items():
for node, cage_info in nodes.items():
cage_location = cage_info["location"]
# note that the requests created here are not registered with
# interfaces and enqueued to a different pool too, they are
# therefore entitled to termination without warning at shutdown,
# this is ok, because they do no useful work for the clients
request = Request(timeout = self._request_timeout,
interface = "__health_monitor__", protocol = "n/a",
parameters = dict(auth_tokens = dict()),
description = "probing cage {0:s} at {1:s}".format(cage, cage_location))
self._probe_thread_pool.enqueue(request, self.wu_probe_cage,
(node, cage, cage_location, cage_info["probe_result"]), {})
# then again yield to polling the queue for a while
if not self._poll_up_down_queue(per_cage_interval):
break
except:
pmnc.log.error(exc_string()) # log and ignore
###################################
# this method merges cages known to the RPC interface with cages
# previously probed and known to be up, such merging is necessary
# because if a cage dies just before its next advertisement broadcast,
# it would disappear from known, but will not be probed again and
# hence thought to be up forever
@staticmethod
def _merge_cages(known_cages: dict, up_cages: dict):
probe_cages = known_cages # merging in place
for up_cage, up_nodes in up_cages.items():
for up_node, up_cage_info in up_nodes.items():
probe_nodes = probe_cages.setdefault(up_cage, {})
if up_node in probe_nodes:
cage_info = probe_nodes[up_node]
if cage_info["location"] == up_cage_info["location"]:
cage_info.update(probe_result = up_cage_info["probe_result"])
else:
cage_info.update(probe_result = "restarted") # note this case
else:
probe_nodes[up_node] = up_cage_info
###################################
# a call to this method is enqueued to a private thread pool
# for each cage to probe on every pass of _probe_thread
def wu_probe_cage(self, node, cage, location, prev_probe_result):
if pmnc.request.expired: # no need to report anything for a probing request
return
if pmnc.log.debug:
pmnc.log.debug("sending probe")
try:
probe_result = self._probe_cage(node, cage, location)
except:
pmnc.log.warning("probe failed: {0:s}".format(exc_string()))
self._up_down_queue.push((node, cage, "down"))
else:
if pmnc.log.debug:
pmnc.log.debug("probe returned successfully")
if prev_probe_result == "restarted": # if the cage has restarted
self._up_down_queue.push((node, cage, "down")) # we push "down" event first
self._up_down_queue.push((node, cage, "up", location, probe_result))
###################################
# this method is invoked by one of the private pool threads
# to send the actual probe call to the cage being probed
@typecheck
def _probe_cage(self, node, cage, location) -> dict:
# health monitor has to create rpc resources manually, not using
# pmnc(cage) syntax, because we need to access exact cage at exact
# node and location (i.e. host and port) and to avoid discovery
connect_timeout = pmnc.config_resource_rpc.get("discovery_timeout")
rpc = pmnc.protocol_rpc.Resource("{0:s}.{1:s}".format(node, cage),
broadcast_address = ("n/a", 0),
discovery_timeout = connect_timeout,
multiple_timeout_allowance = 0.0,
flock_id = "unused",
exact_locations = { cage: location }, # this prevents discovery
pool__resource_name = cage)
rpc.connect()
try:
rpc.begin_transaction("", source_module_name = __name__, transaction_options = {},
resource_args = (), resource_kwargs = {})
try:
probe_result = rpc.health_monitor_event.probe() # there, an RPC call
except:
rpc.rollback()
raise
else:
rpc.commit()
finally:
rpc.disconnect()
return probe_result # if the cage returns anything but a dict, it is considered a failure
###################################
# this method is called by the _probe_thread during its idle times
# to fetch up/down events posted to the _up_down_queue by the probe
# threads and in response to maintain structures such as _up_cages
def _poll_up_down_queue(self, timeout: float) -> bool: # returns "should keep running"
poll_timeout = Timeout(timeout)
while not poll_timeout.expired:
pop_timeout = Timeout(min(poll_timeout.remain, 1.0))
while not pop_timeout.expired:
event = pop_timeout.pop(self._up_down_queue)
if event is not None:
try:
node, cage, up_down, *args = event
if up_down == "up":
location, probe_result = args
# add the cage to cages known to be up and schedule
# application notification call if it was down or
# returned a different probe result
cage_info = self._up_cages.setdefault(cage, {}).setdefault(node, {})
if not cage_info or cage_info["probe_result"] != probe_result:
self._schedule_up_down_event(node, cage, "up", probe_result)
cage_info.update(location = location, probe_result = probe_result)
elif up_down == "down":
# remove the cage from cages known to be up and schedule
# application notification call it was up
if self._up_cages.setdefault(cage, {}).pop(node, None):
self._schedule_up_down_event(node, cage, "down")
except:
pmnc.log.error(exc_string()) # log and ignore
if current_thread().stopped():
return False
return True
###################################
# this method is called by the _probe_thread in response to change
# of some cage's state detected in _poll_up_down_queue
def _schedule_up_down_event(self, node, cage, up_down, probe_result = None):
# application notification invokes methods from health_monitor_event module
# and must be executed just like a regular request from some interface
request = pmnc.interfaces.begin_request(
timeout = self._request_timeout,
interface = "__health_monitor__", protocol = "n/a",
parameters = dict(auth_tokens = dict()),
description = "cage {0:s}.{1:s} is {2:s}".format(node, cage, up_down))
# note that this request is not waited upon
pmnc.interfaces.enqueue(request, self.wu_process_event, (node, cage, up_down, probe_result))
###################################
# this method is invoked by one of the interfaces pool threads to register
# the event of some cage going up or down by calling an appropriate method
# from the health_monitor_event module
@typecheck
def wu_process_event(self, node: str, cage: str, up_down: one_of("up", "down"),
probe_result: optional(dict)):
try:
# see for how long the request was on the execution queue up to this moment
# and whether it has expired in the meantime, if it did there is no reason
# to proceed and we simply bail out
if pmnc.request.expired:
pmnc.log.error("request has expired and will not be processed")
success = False
return # goes through finally section below
with pmnc.performance.request_processing():
self._process_event(node, cage, up_down, probe_result)
except:
pmnc.log.error(exc_string()) # log and ignore
success = False
else:
success = True
finally: # the request ends itself
pmnc.interfaces.end_request(success) # possibly way after deadline
###################################
def _process_event(self, node, cage, up_down, probe_result):
if up_down == "up":
pmnc.health_monitor_event.cage_up(node, cage, probe_result)
elif up_down == "down":
pmnc.health_monitor_event.cage_down(node, cage)
class _SMPPConnection:
@typecheck
def __init__(self, name: str,
in_q: with_attr("push"),
out_q: with_attr("pop"),
inflight: with_attr("add", "remove"),
*,
server_address: (str, int),
connect_timeout: float,
response_timeout: float,
system_id: str,
password: str,
system_type: str,
esme_ton: byte,
esme_npi: byte,
esme_addr: str,
bind_pdu: one_of(BindReceiverPDU, BindTransmitterPDU, BindTransceiverPDU)):
self._name = name
# input and output queues and inflight requests registry are external
# to the connection object and remain persistent across disconnects to
# prevent losing packets
self._in_q = in_q
self._out_q = out_q
self._inflight = inflight
self._server_address = server_address
self._connect_timeout = connect_timeout
self._response_timeout = response_timeout
self._bind_pdu = \
bind_pdu.create(system_id = system_id.encode("ascii", "replace"),
password = password.encode("ascii", "replace"),
system_type = system_type.encode("ascii", "replace"),
interface_version = 0x34,
addr_ton = esme_ton,
addr_npi = esme_npi,
address_range = esme_addr.encode("ascii", "replace"))
self._bound = Event()
self._failed = Event()
failed = property(lambda self: self._failed.is_set())
###################################
def start(self):
self._bind_timeout = Timeout(self._connect_timeout)
self._connect(self._bind_timeout.remain)
self._start_threads()
try:
self._wait_for_bind()
except:
self.stop()
raise
def stop(self):
self._stop_threads()
self._disconnect()
###################################
def _connect(self, timeout):
s = socket(AF_INET, SOCK_STREAM)
try:
s.settimeout(timeout or 0.01)
s.connect(self._server_address)
s.setblocking(False)
except:
s.close()
raise
else:
self._socket = s
def _disconnect(self):
try:
self._socket.close()
except:
pmnc.log(exc_string()) # log and ignore
###################################
def _start_threads(self):
self._reader = HeavyThread(target = self._reader_proc, name = "{0:s}/rd".format(self._name))
self._reader.start()
self._writer = HeavyThread(target = self._writer_proc, name = "{0:s}/wr".format(self._name))
self._writer.start()
def _stop_threads(self):
self._writer.stop() # the writer thread must be stopped first, because it depends
self._reader.stop() # on the reader thread to be active to perform unbind
###################################
def _wait_for_bind(self):
while not self._bind_timeout.expired:
self._bound.wait(min(0.5, self._bind_timeout.remain)) # resort to polling because two
if self._bound.is_set(): # events need to be waited upon
break
if self._failed.is_set():
raise Exception("binding attempt failed")
else:
raise Exception("binding attempt timed out")
###################################
class ThreadStopped(Exception): pass
###################################
def read(self, n):
while len(self._read_data) < n:
ss = select([self._socket], [], [], 1.0)[0]
if current_thread().stopped(): # this particular exception is accounted for
raise _SMPPConnection.ThreadStopped("connection is being closed") # and masked out in _reader_proc below
if not ss: continue
self._read_data += ss[0].recv(4096)
result, self._read_data = self._read_data[:n], self._read_data[n:]
return result
###################################
def _reader_proc(self):
self._read_data = b""
while True:
try:
try:
pdu = PDU.read(self)
except SMPPPDUReadError as e: # mask out the exception from
if isinstance(e.__cause__, _SMPPConnection.ThreadStopped): # the thread being stopped
break # while
else:
raise
if isinstance(pdu, RequestPDU):
if pmnc.log.debug:
pmnc.log.debug("<< [REQ] {0:s}".format(str(pdu)))
self._in_q.push(pdu)
elif isinstance(pdu, ResponsePDU):
req = self._inflight.remove(pdu)
if req is not None:
if pmnc.log.debug:
pmnc.log.debug("<< [RSP] {0:s}".format(str(pdu)))
req.set_response(pdu)
else:
pmnc.log.warning("<< [RSP?!] {0:s}".format(str(pdu)))
else:
pmnc.log.warning("<< [???] {0:s}".format(str(pdu)))
except:
pmnc.log.error(exc_string())
self._failed.set()
break
###################################
def _writer_proc(self):
try:
# perform synchronous bind, using cumulative timeout
if not self._write_pdu(self._bind_pdu, False, self._bind_timeout):
return
_wait_response(self._bind_pdu, self._bind_timeout.remain)
self._bound.set()
while not current_thread().stopped(): # lifetime loop
pdu = self._out_q.pop(1.0)
if pdu and not self._write_pdu(pdu, False, Timeout(self._response_timeout)):
break
# perform synchronous unbind
unbind_pdu = UnbindPDU.create()
unbind_timeout = Timeout(self._response_timeout)
self._write_pdu(unbind_pdu, True, unbind_timeout)
_wait_response(unbind_pdu, unbind_timeout.remain)
except:
pmnc.log.error(exc_string())
self._failed.set()
###################################
def _write_pdu(self, pdu, stopped, timeout):
request_pdu = isinstance(pdu, RequestPDU)
if request_pdu: self._inflight.add(pdu)
try:
self._write_packet(pdu.serialize(), stopped, timeout)
except _SMPPConnection.ThreadStopped:
if request_pdu: self._inflight.remove(pdu)
return False
except:
if request_pdu: self._inflight.remove(pdu)
raise
if pmnc.log.debug:
if request_pdu:
pmnc.log.debug(">> [REQ] {0:s}".format(str(pdu)))
elif isinstance(pdu, ResponsePDU):
pmnc.log.debug(">> [RSP] {0:s}".format(str(pdu)))
else:
pmnc.log.debug(">> [???] {0:s}".format(str(pdu)))
return True
###################################
def _write_packet(self, packet, stopped, timeout):
while packet:
ss = select([], [self._socket], [], min(timeout.remain, 1.0))[1]
if current_thread().stopped() and not stopped:
raise _SMPPConnection.ThreadStopped("connection is being closed")
if timeout.expired:
raise Exception("timeout writing packet")
if ss:
sent = ss[0].send(packet)
packet = packet[sent:]
def _start_threads(self):
self._reader = HeavyThread(target = self._reader_proc, name = "{0:s}/rd".format(self._name))
self._reader.start()
self._writer = HeavyThread(target = self._writer_proc, name = "{0:s}/wr".format(self._name))
self._writer.start()
class Interface: # SMPP interface
@typecheck
def __init__(self, name: str, *,
server_address: (str, int),
connect_timeout: float,
response_timeout: float,
ping_interval: optional(float),
system_id: str,
password: str,
system_type: str,
esme_ton: byte,
esme_npi: byte,
esme_addr: str,
esme_type: one_of("rcvr", "xmit", "xcvr"),
request_timeout: optional(float) = None,
**kwargs): # this kwargs allows for extra application-specific
# settings in config_interface_smpp_X.py
self._name = name
self._response_timeout = response_timeout
if ping_interval:
self._ping_timeout = Timeout(ping_interval)
self._ping_response_timeout = Timeout(response_timeout)
else:
self._ping_timeout = self._ping_response_timeout = None
self._ping_request = None
self._in_q = InterlockedQueue()
self._out_q = InterlockedQueue()
self._inflight = InflightRequests()
self._ceased = Event()
if esme_type == "rcvr":
bind_pdu = BindReceiverPDU
elif esme_type == "xmit":
bind_pdu = BindTransmitterPDU
elif esme_type == "xcvr":
bind_pdu = BindTransceiverPDU
self._create_connection = \
lambda: _SMPPConnection(name, self._in_q, self._out_q, self._inflight,
server_address = server_address,
connect_timeout = connect_timeout,
response_timeout = response_timeout,
system_id = system_id,
password = password,
system_type = system_type,
esme_ton = esme_ton,
esme_npi = esme_npi,
esme_addr = esme_addr,
bind_pdu = bind_pdu)
self._request_timeout = request_timeout or \
pmnc.config_interfaces.get("request_timeout") # this is now static
if pmnc.request.self_test == __name__: # self-test
self._process_request = kwargs["process_request"]
name = property(lambda self: self._name)
ceased = property(lambda self: self._ceased.is_set())
###################################
def start(self):
self._maintainer = HeavyThread(target = self._maintainer_proc, name = "{0:s}/mnt".format(self.name))
self._maintainer.start()
def cease(self):
self._ceased.set()
self._maintainer.stop()
def stop(self):
pass
###################################
def _maintainer_proc(self):
while not current_thread().stopped():
try:
# try to establish a connection, do it infinitely or until the interface is stopped
while True:
try:
self._connection = self._create_connection()
self._connection.start()
except:
pmnc.log.error(exc_string())
failure_timeout = max(self._request_timeout, 30.0)
if current_thread().stopped(failure_timeout):
return
else:
break # while True
try:
while not current_thread().stopped() and not self._connection.failed:
# process incoming PDUs
req = self._in_q.pop(1.0)
if req is not None:
self._handle_pdu(req)
# if there is an outstanding ping request, check for response
if self._ping_request and self._ping_response_timeout.expired:
ping_request, self._ping_request = self._ping_request, None
_wait_response(ping_request, 0.001)
# if it's time to send another ping request, do so
if self._ping_timeout and self._ping_timeout.expired:
try:
self._ping_request = EnquireLinkPDU.create()
self._out_q.push(self._ping_request)
self._ping_response_timeout.reset()
finally:
self._ping_timeout.reset()
finally:
self._connection.stop()
except:
pmnc.log.error(exc_string()) # log and ignore
###################################
# this method processes the request PDUs received by this interface from SMSC
def _handle_pdu(self, req):
if isinstance(req, EnquireLinkPDU): # respond to pings automatically
resp = req.create_response()
self._out_q.push(resp)
else: # note that this interface does not wait for its requests to complete
request = pmnc.interfaces.begin_request(
timeout = self._request_timeout,
interface = self._name, protocol = "smpp",
parameters = dict(auth_tokens = dict()),
description = "incoming {0:s}".format(req))
pmnc.interfaces.enqueue(request, self.wu_handle_pdu, (req, ), {})
###################################
@typecheck
def wu_handle_pdu(self, req: RequestPDU):
try:
# see for how long the request was on the execution queue up to this moment
# and whether it has expired in the meantime, if it did there is no reason
# to proceed and we simply bail out
if pmnc.request.expired:
pmnc.log.error("request has expired and will not be processed")
success = False
return # goes through finally section below
with pmnc.performance.request_processing():
request = dict(pdu = req)
response = dict(pdu = req.create_nack(error_codes.ESME_RUNKNOWNERR))
try:
self._process_request(request, response)
except:
response["pdu"] = req.create_nack(error_codes.ESME_RSYSERR)
raise
finally:
self._out_q.push(response["pdu"])
except:
pmnc.log.error(exc_string()) # log and ignore
success = False
else:
success = True
finally: # the request ends itself
pmnc.interfaces.end_request(success) # possibly way after deadline
###################################
def _process_request(self, request, response):
handler_module_name = "interface_{0:s}".format(self._name)
pmnc.__getattr__(handler_module_name).process_request(request, response)
###################################
# this method is called by the coupled resources to send a request PDU to SMSC
@typecheck
def send(self, req: RequestPDU, timeout: optional(float) = None) -> optional(ResponsePDU):
self._out_q.push(req)
if timeout is not None:
return _wait_response(req, min(timeout, self._response_timeout))
class Interface(AdapterHost): # JMS interface
@typecheck
def __init__(self, name: str, *,
java: os_path.isfile,
arguments: tuple_of(str),
classpath: str,
jndi: dict_of(str, str),
factory: str,
queue: str,
username: str,
password: str,
request_timeout: optional(float) = None,
**kwargs): # this kwargs allows for extra application-specific
# settings in config_interface_jms_X.py
self._name = name
AdapterHost.__init__(self, "org.pythomnic.jms.Receiver",
java = java, arguments = arguments, classpath = classpath,
jndi = jndi, factory = factory, queue = queue,
username = username, password = password)
self._request_timeout = request_timeout or \
pmnc.config_interfaces.get("request_timeout") # this is now static
if pmnc.request.self_test == __name__: # self-test
self._process_request = kwargs["process_request"]
name = property(lambda self: self._name)
###################################
def start(self):
self._maintainer = HeavyThread(target = self._maintainer_proc,
name = "{0:s}:mnt".format(self._name))
self._maintainer.start()
def cease(self):
self._maintainer.stop()
def stop(self):
pass
###################################
# this thread keeps the interface up by (re)starting
# the adapter and its controlling i/o threads
def _maintainer_proc(self):
while not current_thread().stopped():
while True: # keep trying to start the adapter
try:
self._start_adapter("interface", self._name, Timeout(15.0))
except:
pmnc.log.error(exc_string())
failure_timeout = max(self._request_timeout, 30.0)
if current_thread().stopped(failure_timeout):
return
else:
break
# now that the adapter is running, keep receiving messages
# until the adapter exits or the interface is ceased
try:
try:
while self._adapter_running():
# even when the queue is idle, the adapter should be sending in a ping once in 3 seconds
receive_timeout = Timeout(self._request_timeout + 3.0)
pkt = receive_timeout.pop(self._stdout_queue)
if pkt is None:
raise Exception("adapter process failed to produce a message")
if "XPmncError" in pkt: # adapter reports error, abort
raise Exception(pkt["XPmncError"])
request_id = pkt.pop("XPmncRequestID")
if current_thread().stopped(): # any command after shutdown gets an EXIT response
response = Packet(XPmncResponse = "EXIT", XPmncRequestID = request_id)
self._stdin_queue.push(response)
break
request = pkt.pop("XPmncRequest")
if request == "NOOP": # ping always gets an OK response
response = Packet(XPmncResponse = "OK", XPmncRequestID = request_id)
self._stdin_queue.push(response)
elif request == "RECEIVE": # process an incoming message
message_id = pkt["JMSMessageID"]
if message_id not in self._processed_messages: # don't process the message again
success = self._process_message(message_id, pkt, receive_timeout.remain)
else:
success = True
response = Packet(XPmncResponse = success and "COMMIT" or "ROLLBACK",
XPmncRequestID = request_id, XPmncMessageID = message_id)
self._stdin_queue.push(response)
else:
raise Exception("invalid request")
finally:
self._stop_adapter(Timeout(5.0)) # make sure the adapter is not running
except:
pmnc.log.error(exc_string()) # log and ignore
###################################
# request processing in a message-oriented interface is rather straightforward
def _process_message(self, message_id, pkt, request_timeout):
message_text = pkt.pop("XPmncMessageText")
message_description = "JMS message {0:s}{1:s}".format(message_id,
", corr.id {0:s}".format(pkt["JMSCorrelationID"])
if pkt.get("JMSCorrelationID") else "")
request = pmnc.interfaces.begin_request(
timeout = min(request_timeout, self._request_timeout),
interface = self._name, protocol = "jms",
parameters = dict(auth_tokens = dict()),
description = message_description)
try:
pmnc.interfaces.enqueue(request, self.wu_process_request,
(message_id, message_text, pkt.copy())).wait()
except WorkUnitTimedOut:
pmnc.log.error("{0:s} processing timed out".format(message_description))
success = None
except:
pmnc.log.error("{0:s} processing failed: {1:s}".\
format(message_description, exc_string()))
success = False
else:
if pmnc.log.debug:
pmnc.log.debug("{0:s} processing succeeded".format(message_description))
self._processed_messages.add(message_id)
success = True
finally:
pmnc.interfaces.end_request(success, request)
return success == True
###################################
# this method is a work unit executed by one of the interface pool threads
# if this method fails, the exception is rethrown in _process_message in wait()
@typecheck
def wu_process_request(self, message_id: str, message_text: str, headers: dict):
# see for how long the request was on the execution queue up to this moment
# and whether it has expired in the meantime, if it did there is no reason
# to proceed and we simply bail out
if pmnc.request.expired:
pmnc.log.error("request has expired and will not be processed")
return
try:
with pmnc.performance.request_processing():
request = dict(message_id = message_id, message_text = message_text, headers = headers)
self._process_request(request, {})
except:
pmnc.log.error(exc_string()) # don't allow an exception to be silenced
raise # when this work unit is not waited upon
###################################
def _process_request(self, request, response):
handler_module_name = "interface_{0:s}".format(self._name)
pmnc.__getattr__(handler_module_name).process_request(request, response)
def start(self):
self._maintainer = HeavyThread(target = self._maintainer_proc,
name = "{0:s}:mnt".format(self._name))
self._maintainer.start()
def start(self):
self._scheduler = HeavyThread(target=self._scheduler_proc,
name="{0:s}:sch".format(self._name))
self._scheduler.start()
def test_execute_success():
fake_request(10.0)
request_id = None
response = None
def caller(*args, **kwargs):
fake_request(6.0)
nonlocal request_id, response
request_id = pmnc.request.unique_id
pmnc.request.parameters["AAA"] = "BBB"
pmnc.request.describe("my request")
response = pmnc.__getattr__(__name__).execute_reverse(
"good_cage", "module", "method", args, kwargs)
assert "good_cage" not in _rq_queues
th = HeavyThread(target=caller, args=(1, "foo"), kwargs={"biz": "baz"})
th.start()
try:
sleep(2.0)
assert "good_cage" in _rq_queues
assert request_id in _rs_queues
req_id, req = _rq_queues["good_cage"].pop()
assert req_id == request_id
assert abs(req["request"].pop("deadline") - time() - 4.0) < 1.0
assert req == dict \
(
source_cage = __cage__,
target_cage = "good_cage",
module = "module",
method = "method",
args = (1, "foo"),
kwargs = { "biz": "baz" },
request = dict(protocol = pmnc.request.protocol,
interface = pmnc.request.interface,
unique_id = request_id,
description = "my request",
parameters = dict(auth_tokens = {}, AAA = "BBB"),
log_levels = []),
)
_rs_queues[request_id].push({"result": "RESULT"})
sleep(2.0)
assert "good_cage" in _rq_queues
assert request_id not in _rs_queues
finally:
th.stop()
assert response == "RESULT"
assert "good_cage" in _rq_queues
assert request_id not in _rs_queues
def start():
global _perf_thread
_perf_thread = HeavyThread(target = _perf_thread_proc, name = "performance")
_perf_thread.start()
class Interface: # file-loading interface
@typecheck
def __init__(self, name: str, *,
source_directory: os_path.isdir,
filename_regex: str,
interval: float,
request_timeout: optional(float) = None,
**kwargs): # this kwargs allows for extra application-specific
# settings in config_interface_file_X.py
self._name = name
self._source_directory = source_directory
self._valid_filename = by_regex("^{0:s}$".format(filename_regex))
self._interval = interval
self._request_timeout = request_timeout or \
pmnc.config_interfaces.get("request_timeout") # this is now static
# this set contains names of files that have been processed
# but not deleted due to filesystem deletion failure
self._processed_files = set()
if pmnc.request.self_test == __name__: # self-test
self._process_request = kwargs["process_request"]
name = property(lambda self: self._name)
###################################
def start(self):
self._spooler = HeavyThread(target = self._spooler_proc,
name = "{0:s}:spool".format(self._name))
self._spooler.start()
def cease(self):
self._spooler.stop()
def stop(self):
pass
###################################
# this thread periodically scans the configured directory for files
def _spooler_proc(self):
interval = self._interval
while not current_thread().stopped(interval):
try:
# enumerate matching files in the source directory
file_names = [ os_path.join(self._source_directory, file_name)
for file_name in os.listdir(self._source_directory)
if self._valid_filename(file_name) and
os_path.isfile(os_path.join(self._source_directory, file_name)) ]
if not file_names:
interval = self._interval # no files, rescan after a delay
continue
# sort and process the file names in alphabetical order
file_names.sort()
for file_name in file_names:
if current_thread().stopped(): # processing of all files may take a long time
break # for
elif file_name in self._processed_files: # just remove the already processed file
self._remove_file(file_name)
elif self._process_file(file_name): # remove the file if it has been processed
self._remove_file(file_name)
else:
interval = self._interval # processing of some file failed,
break # for # rescan after a delay
else: # all the files have been processed, rescan immediately,
interval = 0.0 # hoping for the new files to appear in the meantime
except:
pmnc.log.error(exc_string())
interval = self._interval
###################################
def _remove_file(self, file_name):
try:
_retry_remove_file(file_name, 2.0)
except:
pmnc.log.warning(exc_string())
else:
self._processed_files.remove(file_name)
###################################
def _process_file(self, file_name):
request = pmnc.interfaces.begin_request(
timeout = self._request_timeout,
interface = self._name, protocol = "file",
parameters = dict(auth_tokens = dict()),
description = "file {0:s}".format(file_name))
# enqueue the request and wait for its completion
try:
pmnc.interfaces.enqueue(request, self.wu_process_request,
(file_name, )).wait()
except WorkUnitTimedOut:
pmnc.log.error("file processing timed out")
success = None
except:
pmnc.log.error("file processing failed: {0:s}".format(exc_string()))
success = False
else:
if pmnc.log.debug:
pmnc.log.debug("file processing succeeded")
self._processed_files.add(file_name)
success = True
finally:
pmnc.interfaces.end_request(success, request)
return success == True
###################################
# this method is a work unit executed by one of the interface pool threads
# if this method fails, the exception is rethrown in _process_file in wait()
@typecheck
def wu_process_request(self, file_name: str):
# see for how long the request was on the execution queue up to this moment
# and whether it has expired in the meantime, if it did there is no reason
# to proceed and we simply bail out
if pmnc.request.expired:
pmnc.log.error("request has expired and will not be processed")
return
try:
with pmnc.performance.request_processing():
request = dict(file_name = os_path.join(self._source_directory, file_name))
self._process_request(request, {})
except:
pmnc.log.error(exc_string()) # don't allow an exception to be silenced
raise # when this work unit is not waited upon
###################################
def _process_request(self, request, response):
handler_module_name = "interface_{0:s}".format(self._name)
pmnc.__getattr__(handler_module_name).process_request(request, response)
def secondary_startup(node, cage, mode):
cage_dir = os_path.join(cages_dir, cage)
logs_dir = os_path.join(cage_dir, "logs")
lib_dir = os_path.join(cage_dir, "lib")
sys_path.insert(0, lib_dir)
log_abbrevs = {
1: "ERR",
2: "MSG",
3: "WRN",
4: "LOG",
5: "INF",
6: "DBG",
7: "NSE"
}
log_encoding = "windows-1251"
log_translate = b" \t " + bytes(range(
32, 256))
log_lock = Lock()
log_yyyymmdd = None
log_file = None
# the following function will serve for all cage's logging
def log(message, *, msg_level):
nonlocal log_yyyymmdd, log_file
line_time = time()
line_yyyymmdd, line_hhmmss = strftime("%Y%m%d %H:%M:%S",
localtime(line_time)).split(" ")
log_line = "{0:s}.{1:02d} {2:s} [{3:s}] {4:s}".format(
line_hhmmss,
int(line_time * 100) % 100, log_abbrevs.get(msg_level, "???"),
current_thread().name, message)
with log_lock:
if line_yyyymmdd != log_yyyymmdd: # rotate log file
try:
new_log_file_name = os_path.join(
logs_dir,
"{0:s}-{1:s}.log".format(cage, line_yyyymmdd))
new_log_file = fopen(
new_log_file_name,
os.O_WRONLY | os.O_CREAT | os.O_APPEND)
except:
pass # if rotation fails, previous log file will still be used
else:
try:
close(log_file)
except:
pass # this also catches the attempt to close None
log_file = new_log_file
log_yyyymmdd = line_yyyymmdd
if log_file is not None:
if message:
write(
log_file,
log_line.encode(log_encoding,
"replace").translate(log_translate) +
b"\n")
if msg_level == 1:
fsync(log_file)
###################################
# create loader instance using initial default logging level
pmnc = ModuleLoader(node, cage, cage_dir, log, "LOG", 2.0, 1.0)
###################################
current_thread().name = "startup"
if mode == "NORMAL":
log("the cage is starting up", msg_level=2)
elif mode == "FAILURE":
log("the cage is restarting after a failure", msg_level=2)
###################################
if win32_com:
_main_thread_id = GetCurrentThreadId()
def cage_thread_proc():
try:
pmnc.startup.start()
try:
while not pmnc.startup.wait(3.0):
pmnc.startup.maintenance()
except:
pmnc.log.error(exc_string()) # log and ignore
finally:
pmnc.startup.stop()
finally:
if win32_com: # release the main thread blocked in PumpMessages
PostThreadMessage(_main_thread_id, WM_QUIT)
cage_thread = HeavyThread(target=cage_thread_proc, name="cage")
cage_thread.start()
###################################
def termination_watchdog_proc():
try:
while stdout.write("\n") > 0:
stdout.flush()
sleep(3.0)
except:
pass
finally:
pmnc.startup.exit()
termination_watchdog = HeavyThread(target=termination_watchdog_proc,
name="stdout")
termination_watchdog.start()
###################################
# wait for the cage thread to detect shutdown and terminate
if win32_com:
PumpMessages() # in the meanwhile become a message pump
cage_thread.join()
###################################
log("the cage has been properly shut down", msg_level=2)
log("", msg_level=1) # force flush of a log file
class Interface:
@typecheck
def __init__(self, name: str, *,
listener_address: (str, int),
request_timeout: optional(float) = None,
**kwargs): # this kwargs allows for extra application-specific
# settings in config_interface_udp_X.py
self._name = name
self._listener_address = listener_address
self._request_timeout = request_timeout or \
pmnc.config_interfaces.get("request_timeout") # this is now static
if pmnc.request.self_test == __name__: # self-test
self._process_request = kwargs["process_request"]
name = property(lambda self: self._name)
listener_address = property(lambda self: self._listener_address)
###################################
def start(self):
self._listener = HeavyThread(target = self._listener_proc,
name = "{0:s}:lsn".format(self._name))
self._listener.start()
###################################
def cease(self):
self._listener.stop()
###################################
def stop(self):
pass
###################################
def _create_server_socket(self) -> socket:
s = socket(AF_INET, SOCK_DGRAM)
try:
s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
if have_reuse_port:
s.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
s.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
s.bind(self._listener_address)
except:
s.close()
raise
else:
return s
###################################
def _listener_proc(self):
while not current_thread().stopped():
try:
s = self._create_server_socket()
try:
pmnc.log.message("started listening for packets at {0[0]:s}:{0[1]:d}".\
format(self._listener_address))
while not current_thread().stopped():
if not select([s], [], [], 1.0)[0]:
continue
try:
packet_b, (client_addr, client_port) = s.recvfrom(57344)
except socket_timeout:
continue
except socket_error as e:
if e.args[0] == 10054: # workaround for this issue: http://support.microsoft.com/kb/263823
continue
else:
raise
if pmnc.log.debug:
pmnc.log.debug("incoming UDP packet from {0:s}:{1:d}, {2:d} byte(s)".\
format(client_addr, client_port, len(packet_b)))
# the listener thread initiates the received packet processing
try:
self._enqueue_packet(packet_b, client_addr, client_port)
except:
pmnc.log.error(exc_string()) # log and ignore
finally:
s.close()
pmnc.log.message("stopped listening")
except:
pmnc.log.error(exc_string()) # log and ignore
###################################
def _enqueue_packet(self, packet_b, client_addr, client_port):
# create a new request for processing the message
request = pmnc.interfaces.begin_request(
timeout = self._request_timeout,
interface = self._name, protocol = "udp",
parameters = dict(auth_tokens = dict(peer_ip = client_addr)),
description = "UDP packet from {0:s}, {1:d} byte(s)".\
format(client_addr, len(packet_b)))
# enqueue the request but do not wait for its completion
pmnc.interfaces.enqueue(request, self.wu_process_request, (packet_b, ))
###################################
def wu_process_request(self, packet_b):
try:
# see for how long the request was on the execution queue up to this moment
# and whether it has expired in the meantime, if it did there is no reason
# to proceed and we simply bail out
if pmnc.request.expired:
pmnc.log.error("request has expired and will not be processed")
success = False
return
with pmnc.performance.request_processing():
request = dict(packet = packet_b)
self._process_request(request, {})
except:
pmnc.log.error(exc_string())
success = False
else:
success = True
finally: # the request ends itself
pmnc.interfaces.end_request(success) # possibly way after deadline
###################################
def _process_request(self, request, response):
handler_module_name = "interface_{0:s}".format(self._name)
pmnc.__getattr__(handler_module_name).process_request(request, response)
class Interface: # schedule interface
@typecheck
def __init__(
self,
name: str,
*,
format: str,
match: str,
request_timeout: optional(float) = None,
**kwargs): # this kwargs allows for extra application-specific
# settings in config_interface_schedule_X.py
self._name = name
self._format = format
self._match = by_regex("^{0:s}$".format(match))
self._request_timeout = request_timeout or \
pmnc.config_interfaces.get("request_timeout") # this is now static
self._last_tick = Interface._get_current_tick()
if pmnc.request.self_test == __name__: # self-test
self._process_request = kwargs["process_request"]
name = property(lambda self: self._name)
###################################
def start(self):
self._scheduler = HeavyThread(target=self._scheduler_proc,
name="{0:s}:sch".format(self._name))
self._scheduler.start()
def cease(self):
self._scheduler.stop()
def stop(self):
pass
###################################
# this thread keeps track of time and initiates requests as appropriate
def _scheduler_proc(self):
while not current_thread().stopped(1.1 - modf(time())[0]):
try:
current_tick = Interface._get_current_tick()
try:
for tick in range(self._last_tick + 1, current_tick + 1):
invocation_time = datetime.fromtimestamp(tick)
if self._match(invocation_time.strftime(self._format)):
try:
self._fire_request(invocation_time)
except:
pmnc.log.error(exc_string()) # log and ignore
finally:
self._last_tick = tick
except:
pmnc.log.error(exc_string()) # log and ignore
###################################
def _fire_request(self, invocation_time):
request = pmnc.interfaces.begin_request(
timeout=self._request_timeout,
interface=self._name,
protocol="schedule",
parameters=dict(auth_tokens=dict()),
description=invocation_time.strftime("at %Y-%m-%d %H:%M:%S"))
# note that this interface does not wait for its requests to complete
pmnc.interfaces.enqueue(request, self.wu_process_request,
(invocation_time, ), {})
###################################
@typecheck
def wu_process_request(self, invocation_time: datetime):
try:
# see for how long the request was on the execution queue up to this moment
# and whether it has expired in the meantime, if it did there is no reason
# to proceed and we simply bail out
if pmnc.request.expired:
pmnc.log.error("request has expired and will not be processed")
success = False
return # goes through finally section below
with pmnc.performance.request_processing():
request = dict(invocation_time=invocation_time)
self._process_request(request, {})
except:
pmnc.log.error(exc_string()) # log and ignore
success = False
else:
success = True
finally: # the request ends itself
pmnc.interfaces.end_request(success) # possibly way after deadline
###################################
def _process_request(self, request, response):
handler_module_name = "interface_{0:s}".format(self._name)
pmnc.__getattr__(handler_module_name).process_request(
request, response)
###################################
@staticmethod
def _get_current_tick():
return int(mktime(datetime.now().timetuple()))
def start(self):
self._listener = HeavyThread(target = self._listener_proc,
name = "{0:s}:lsn".format(self._name))
self._listener.start()
def start():
global _perf_thread
_perf_thread = HeavyThread(target = _perf_thread_proc, name = "performance")
_perf_thread.start()
def test_cross_db_deadlock():
fake_request(30.0)
db1 = pmnc.state.get_database("db1")
db2 = pmnc.state.get_queue("db2", re_len=6144)
def f(txn):
db1.put(b"key", b"value_1", txn)
Timeout(3.0).wait()
db2.append(pickle("item_1"), txn)
def g(txn):
db2.append(pickle("item_2"), txn)
Timeout(3.0).wait()
db1.put(b"key", b"value_2", txn)
th_f = HeavyThread(target=lambda: pmnc.state.implicit_transaction(f))
th_g = HeavyThread(target=lambda: pmnc.state.implicit_transaction(g))
th_f.start()
th_g.start()
th_f.join()
th_g.join()
# now see what went through
def fetch_result(txn):
value = db1.get(b"key", None, txn)
item1 = _pop(txn, db2, None, unpickle)
item2 = _pop(txn, db2, None, unpickle)
return value, item1, item2
value, item1, item2 = pmnc.state.implicit_transaction(fetch_result)
assert value in (b"value_1", b"value_2")
assert (item1, item2) in (("item_1", "item_2"), ("item_2", "item_1"))
