def test_execute_success(): fake_request(10.0) request_id = None response = None def caller(*args, **kwargs): fake_request(6.0) nonlocal request_id, response request_id = pmnc.request.unique_id pmnc.request.parameters["AAA"] = "BBB" pmnc.request.describe("my request") response = pmnc.__getattr__(__name__).execute_reverse("good_cage", "module", "method", args, kwargs) assert "good_cage" not in _rq_queues th = HeavyThread(target = caller, args = (1, "foo"), kwargs = { "biz": "baz" }) th.start() try: sleep(2.0) assert "good_cage" in _rq_queues assert request_id in _rs_queues req_id, req = _rq_queues["good_cage"].pop() assert req_id == request_id assert abs(req["request"].pop("deadline") - time() - 4.0) < 1.0 assert req == dict \ ( source_cage = __cage__, target_cage = "good_cage", module = "module", method = "method", args = (1, "foo"), kwargs = { "biz": "baz" }, request = dict(protocol = pmnc.request.protocol, interface = pmnc.request.interface, unique_id = request_id, description = "my request", parameters = dict(auth_tokens = {}, AAA = "BBB")), ) _rs_queues[request_id].push({ "result": "RESULT" }) sleep(2.0) assert "good_cage" in _rq_queues assert request_id not in _rs_queues finally: th.stop() assert response == "RESULT" assert "good_cage" in _rq_queues assert request_id not in _rs_queues
def test_WorkSources_end_work(): current_thread().stopped = lambda: False wss = WorkSources(2, 30.0) wss.add_work(1) assert wss.begin_work(0.0) == (False, 1) wss.add_work(0) assert wss.begin_work(0.0) == (False, 0) assert wss.begin_work(0.0) == (False, None) wss.add_work(1) assert wss.begin_work(0.0) == (False, None) th_started = Event() th_got_work = Event() def th_proc(): th_started.set() assert wss.begin_work(10.0) == (False, 1) th_got_work.set() th = HeavyThread(target = th_proc) th.start() th_started.wait() t = Timeout(30.0) wss.end_work(1) th_got_work.wait() assert t.remain > 29.0, t.remain th.stop()
def start(): global _state_thread if not _no_bssdb3: _state_thread = HeavyThread(target=_state_thread_proc, name="state") _state_thread.start() else: pmnc.log.message("module bsddb3 could not be imported, " "persistent state is disabled")
def test_WorkSources_end_work(): current_thread().stopped = lambda: False wss = WorkSources(2, 30.0) wss.add_work(1) assert wss.begin_work(0.0) == (False, 1) wss.add_work(0) assert wss.begin_work(0.0) == (False, 0) assert wss.begin_work(0.0) == (False, None) wss.add_work(1) assert wss.begin_work(0.0) == (False, None) th_started = Event() th_got_work = Event() def th_proc(): th_started.set() assert wss.begin_work(10.0) == (False, 1) th_got_work.set() th = HeavyThread(target=th_proc) th.start() th_started.wait() t = Timeout(30.0) wss.end_work(1) th_got_work.wait() assert t.remain > 29.0, t.remain th.stop()
def start(self): # a separate thread is started to poll each source cage self._pollers = [] for source_cage in self._source_cages: poller = HeavyThread(target = self._poller_proc, args = (source_cage, ), name = "{0:s}:{1:s}".format(self._name, source_cage)) self._pollers.append(poller) for poller in self._pollers: poller.start()
def start(): global _state_thread if not _no_bssdb3: with _state_thread_lock: if not _state_thread: _state_thread = HeavyThread(target=_state_thread_proc, name="state") _state_thread.start() else: pmnc.log.message("module bsddb3 could not be imported, " "persistent state is disabled")
def start(self): # a separate thread is started to poll each source cage self._pollers = [] for source_cage in self._source_cages: poller = HeavyThread(target=self._poller_proc, args=(source_cage, ), name="{0:s}:{1:s}".format( self._name, source_cage)) self._pollers.append(poller) for poller in self._pollers: poller.start()
def test_WorkSources_stop_thread(): th_started = Event() def th_proc(): wss = WorkSources(1, 30.0) th_started.set() assert wss.begin_work(10.0) == (True, None) th = HeavyThread(target = th_proc) th.start() t = Timeout(30.0) th_started.wait() sleep(1.0) th.stop() assert t.remain > 25.0
def test_WorkSources_stop_thread(): th_started = Event() def th_proc(): wss = WorkSources(1, 30.0) th_started.set() assert wss.begin_work(10.0) == (True, None) th = HeavyThread(target=th_proc) th.start() t = Timeout(30.0) th_started.wait() sleep(1.0) th.stop() assert t.remain > 25.0
def test_WorkSources_signal_kick(): th_started = Event() th_got_work = Event() wss = WorkSources(1, 30.0) def th_proc(): th_started.set() assert wss.begin_work(10.0) == (False, 0) th_got_work.set() th = HeavyThread(target = th_proc) th.start() th_started.wait() t = Timeout(30.0) wss.add_work(0) th_got_work.wait() assert t.remain > 29.0 th.stop()
def test_WorkSources_signal_kick(): th_started = Event() th_got_work = Event() wss = WorkSources(1, 30.0) def th_proc(): th_started.set() assert wss.begin_work(10.0) == (False, 0) th_got_work.set() th = HeavyThread(target=th_proc) th.start() th_started.wait() t = Timeout(30.0) wss.add_work(0) th_got_work.wait() assert t.remain > 29.0 th.stop()
def test_cross_db_deadlock(): fake_request(30.0) db1 = pmnc.state.get_database("db1") db2 = pmnc.state.get_queue("db2", re_len=6144) def f(txn): db1.put(b"key", b"value_1", txn) Timeout(3.0).wait() db2.append(pickle("item_1"), txn) def g(txn): db2.append(pickle("item_2"), txn) Timeout(3.0).wait() db1.put(b"key", b"value_2", txn) th_f = HeavyThread(target=lambda: pmnc.state.implicit_transaction(f)) th_g = HeavyThread(target=lambda: pmnc.state.implicit_transaction(g)) th_f.start() th_g.start() th_f.join() th_g.join() # now see what went through def fetch_result(txn): value = db1.get(b"key", None, txn) item1 = _pop(txn, db2, None, unpickle) item2 = _pop(txn, db2, None, unpickle) return value, item1, item2 value, item1, item2 = pmnc.state.implicit_transaction(fetch_result) assert value in (b"value_1", b"value_2") assert (item1, item2) in (("item_1", "item_2"), ("item_2", "item_1"))
def start(): global _perf_thread _perf_thread = HeavyThread(target = _perf_thread_proc, name = "performance") _perf_thread.start()
class Interface: # schedule interface @typecheck def __init__( self, name: str, *, format: str, match: str, request_timeout: optional(float) = None, **kwargs): # this kwargs allows for extra application-specific # settings in config_interface_schedule_X.py self._name = name self._format = format self._match = by_regex("^{0:s}$".format(match)) self._request_timeout = request_timeout or \ pmnc.config_interfaces.get("request_timeout") # this is now static self._last_tick = Interface._get_current_tick() if pmnc.request.self_test == __name__: # self-test self._process_request = kwargs["process_request"] name = property(lambda self: self._name) ################################### def start(self): self._scheduler = HeavyThread(target=self._scheduler_proc, name="{0:s}:sch".format(self._name)) self._scheduler.start() def cease(self): self._scheduler.stop() def stop(self): pass ################################### # this thread keeps track of time and initiates requests as appropriate def _scheduler_proc(self): while not current_thread().stopped(1.1 - modf(time())[0]): try: current_tick = Interface._get_current_tick() try: for tick in range(self._last_tick + 1, current_tick + 1): invocation_time = datetime.fromtimestamp(tick) if self._match(invocation_time.strftime(self._format)): try: self._fire_request(invocation_time) except: pmnc.log.error(exc_string()) # log and ignore finally: self._last_tick = tick except: pmnc.log.error(exc_string()) # log and ignore ################################### def _fire_request(self, invocation_time): request = pmnc.interfaces.begin_request( timeout=self._request_timeout, interface=self._name, protocol="schedule", parameters=dict(auth_tokens=dict()), description=invocation_time.strftime("at %Y-%m-%d %H:%M:%S")) # note that this interface does not wait for its requests to complete pmnc.interfaces.enqueue(request, self.wu_process_request, (invocation_time, ), {}) ################################### @typecheck def wu_process_request(self, invocation_time: datetime): try: # see for how long the request was on the execution queue up to this moment # and whether it has expired in the meantime, if it did there is no reason # to proceed and we simply bail out if pmnc.request.expired: pmnc.log.error("request has expired and will not be processed") success = False return # goes through finally section below with pmnc.performance.request_processing(): request = dict(invocation_time=invocation_time) self._process_request(request, {}) except: pmnc.log.error(exc_string()) # log and ignore success = False else: success = True finally: # the request ends itself pmnc.interfaces.end_request(success) # possibly way after deadline ################################### def _process_request(self, request, response): handler_module_name = "interface_{0:s}".format(self._name) pmnc.__getattr__(handler_module_name).process_request( request, response) ################################### @staticmethod def _get_current_tick(): return int(mktime(datetime.now().timetuple()))
def test_execute_success(): fake_request(10.0) request_id = None response = None def caller(*args, **kwargs): fake_request(6.0) nonlocal request_id, response request_id = pmnc.request.unique_id pmnc.request.parameters["AAA"] = "BBB" pmnc.request.describe("my request") response = pmnc.__getattr__(__name__).execute_reverse( "good_cage", "module", "method", args, kwargs) assert "good_cage" not in _rq_queues th = HeavyThread(target=caller, args=(1, "foo"), kwargs={"biz": "baz"}) th.start() try: sleep(2.0) assert "good_cage" in _rq_queues assert request_id in _rs_queues req_id, req = _rq_queues["good_cage"].pop() assert req_id == request_id assert abs(req["request"].pop("deadline") - time() - 4.0) < 1.0 assert req == dict \ ( source_cage = __cage__, target_cage = "good_cage", module = "module", method = "method", args = (1, "foo"), kwargs = { "biz": "baz" }, request = dict(protocol = pmnc.request.protocol, interface = pmnc.request.interface, unique_id = request_id, description = "my request", parameters = dict(auth_tokens = {}, AAA = "BBB"), log_levels = []), ) _rs_queues[request_id].push({"result": "RESULT"}) sleep(2.0) assert "good_cage" in _rq_queues assert request_id not in _rs_queues finally: th.stop() assert response == "RESULT" assert "good_cage" in _rq_queues assert request_id not in _rs_queues
def secondary_startup(node, cage, mode): cage_dir = os_path.join(cages_dir, cage) logs_dir = os_path.join(cage_dir, "logs") lib_dir = os_path.join(cage_dir, "lib") sys_path.insert(0, lib_dir) log_abbrevs = { 1: "ERR", 2: "MSG", 3: "WRN", 4: "LOG", 5: "INF", 6: "DBG", 7: "NSE" } log_encoding = "windows-1251" log_translate = b" \t " + bytes(range( 32, 256)) log_lock = Lock() log_yyyymmdd = None log_file = None # the following function will serve for all cage's logging def log(message, *, msg_level): nonlocal log_yyyymmdd, log_file line_time = time() line_yyyymmdd, line_hhmmss = strftime("%Y%m%d %H:%M:%S", localtime(line_time)).split(" ") log_line = "{0:s}.{1:02d} {2:s} [{3:s}] {4:s}".format( line_hhmmss, int(line_time * 100) % 100, log_abbrevs.get(msg_level, "???"), current_thread().name, message) with log_lock: if line_yyyymmdd != log_yyyymmdd: # rotate log file try: new_log_file_name = os_path.join( logs_dir, "{0:s}-{1:s}.log".format(cage, line_yyyymmdd)) new_log_file = fopen( new_log_file_name, os.O_WRONLY | os.O_CREAT | os.O_APPEND) except: pass # if rotation fails, previous log file will still be used else: try: close(log_file) except: pass # this also catches the attempt to close None log_file = new_log_file log_yyyymmdd = line_yyyymmdd if log_file is not None: if message: write( log_file, log_line.encode(log_encoding, "replace").translate(log_translate) + b"\n") if msg_level == 1: fsync(log_file) ################################### # create loader instance using initial default logging level pmnc = ModuleLoader(node, cage, cage_dir, log, "LOG", 2.0, 1.0) ################################### current_thread().name = "startup" if mode == "NORMAL": log("the cage is starting up", msg_level=2) elif mode == "FAILURE": log("the cage is restarting after a failure", msg_level=2) ################################### if win32_com: _main_thread_id = GetCurrentThreadId() def cage_thread_proc(): try: pmnc.startup.start() try: while not pmnc.startup.wait(3.0): pmnc.startup.maintenance() except: pmnc.log.error(exc_string()) # log and ignore finally: pmnc.startup.stop() finally: if win32_com: # release the main thread blocked in PumpMessages PostThreadMessage(_main_thread_id, WM_QUIT) cage_thread = HeavyThread(target=cage_thread_proc, name="cage") cage_thread.start() ################################### def termination_watchdog_proc(): try: while stdout.write("\n") > 0: stdout.flush() sleep(3.0) except: pass finally: pmnc.startup.exit() termination_watchdog = HeavyThread(target=termination_watchdog_proc, name="stdout") termination_watchdog.start() ################################### # wait for the cage thread to detect shutdown and terminate if win32_com: PumpMessages() # in the meanwhile become a message pump cage_thread.join() ################################### log("the cage has been properly shut down", msg_level=2) log("", msg_level=1) # force flush of a log file
class HealthMonitor: def __init__(self, **kwargs): self._rpc_interface = pmnc.interfaces.get_interface("rpc") if self._rpc_interface is None: raise Exception("health monitor requires enabled rpc interface") self._probe_thread_pool = pmnc.shared_pools.get_private_thread_pool() self._up_cages = {} # { cage: { node: { location: ..., probe_result: ... } } } self._up_down_queue = InterlockedQueue() self._request_timeout = pmnc.config_interfaces.get("request_timeout") # this is now static if pmnc.request.self_test == __name__: # self-test self._process_event = kwargs["process_event"] self._probe_cage = kwargs["probe_cage"] ################################### def start(self): self._probe_thread = HeavyThread( target=self._probe_thread_proc, name="health_monitor:probe" ) # always called "health_monitor" self._probe_thread.start() def stop(self): self._probe_thread.stop() ################################### # this method is executed in a private thread and is scheduling probe calls # to cages known to the RPC interface or previously probed and found to be up def _probe_thread_proc(self): per_cage_interval = 0.0 # calls to _poll_up_down_queue are interleaved and allow this thread # to maintain structures such as _up_cages in response to events # posted by the probe threads to the _up_down_queue while self._poll_up_down_queue(per_cage_interval): try: # extract all cages currently known to the rpc interface and # merge them with cages previously probed and found to be up, # except for the health_monitor cage itself should be skipped probe_cages = { known_cage: { known_node: dict(location=known_location, probe_result=None) for known_node, known_location in self._rpc_interface.get_nodes(known_cage).items() } for known_cage in self._rpc_interface.get_cages() if known_cage != "health_monitor" } self._merge_cages(probe_cages, self._up_cages) probe_period = pmnc.config.get("probe_period") per_cage_interval = probe_period / (len(probe_cages) + 1) # walk through all cages to be probed and schedule calls to probe # to a private thread pool using fake unregistered requests for cage, nodes in probe_cages.items(): for node, cage_info in nodes.items(): cage_location = cage_info["location"] # note that the requests created here are not registered with # interfaces and enqueued to a different pool too, they are # therefore entitled to termination without warning at shutdown, # this is ok, because they do no useful work for the clients request = Request( timeout=self._request_timeout, interface="__health_monitor__", protocol="n/a", parameters=dict(auth_tokens=dict()), description="probing cage {0:s} at {1:s}".format(cage, cage_location), ) self._probe_thread_pool.enqueue( request, self.wu_probe_cage, (node, cage, cage_location, cage_info["probe_result"]), {} ) # then again yield to polling the queue for a while if not self._poll_up_down_queue(per_cage_interval): break except: pmnc.log.error(exc_string()) # log and ignore ################################### # this method merges cages known to the RPC interface with cages # previously probed and known to be up, such merging is necessary # because if a cage dies just before its next advertisement broadcast, # it would disappear from known, but will not be probed again and # hence thought to be up forever @staticmethod def _merge_cages(known_cages: dict, up_cages: dict): probe_cages = known_cages # merging in place for up_cage, up_nodes in up_cages.items(): for up_node, up_cage_info in up_nodes.items(): probe_nodes = probe_cages.setdefault(up_cage, {}) if up_node in probe_nodes: cage_info = probe_nodes[up_node] if cage_info["location"] == up_cage_info["location"]: cage_info.update(probe_result=up_cage_info["probe_result"]) else: cage_info.update(probe_result="restarted") # note this case else: probe_nodes[up_node] = up_cage_info ################################### # a call to this method is enqueued to a private thread pool # for each cage to probe on every pass of _probe_thread def wu_probe_cage(self, node, cage, location, prev_probe_result): if pmnc.request.expired: # no need to report anything for a probing request return pmnc.log.debug("sending probe") try: probe_result = self._probe_cage(node, cage, location) except: pmnc.log.warning("probe failed: {0:s}".format(exc_string())) self._up_down_queue.push((node, cage, "down")) else: pmnc.log.debug("probe returned successfully") if prev_probe_result == "restarted": # if the cage has restarted self._up_down_queue.push((node, cage, "down")) # we push "down" event first self._up_down_queue.push((node, cage, "up", location, probe_result)) ################################### # this method is invoked by one of the private pool threads # to send the actual probe call to the cage being probed @typecheck def _probe_cage(self, node, cage, location) -> dict: # health monitor has to create rpc resources manually, not using # pmnc(cage) syntax, because we need to access exact cage at exact # node and location (i.e. host and port) and to avoid discovery connect_timeout = pmnc.config_resource_rpc.get("discovery_timeout") rpc = pmnc.protocol_rpc.Resource( "{0:s}.{1:s}".format(node, cage), broadcast_address=("n/a", 0), discovery_timeout=connect_timeout, multiple_timeout_allowance=0.0, flock_id="unused", exact_locations={cage: location}, # this prevents discovery pool__resource_name=cage, ) rpc.connect() try: rpc.begin_transaction( "", source_module_name=__name__, transaction_options={}, resource_args=(), resource_kwargs={} ) try: probe_result = rpc.health_monitor_event.probe() # there, an RPC call except: rpc.rollback() raise else: rpc.commit() finally: rpc.disconnect() return probe_result # if the cage returns anything but a dict, it is considered a failure ################################### # this method is called by the _probe_thread during its idle times # to fetch up/down events posted to the _up_down_queue by the probe # threads and in response to maintain structures such as _up_cages def _poll_up_down_queue(self, timeout: float) -> bool: # returns "should keep running" poll_timeout = Timeout(timeout) while not poll_timeout.expired: pop_timeout = Timeout(min(poll_timeout.remain, 1.0)) while not pop_timeout.expired: event = pop_timeout.pop(self._up_down_queue) if event is not None: try: node, cage, up_down, *args = event if up_down == "up": location, probe_result = args # add the cage to cages known to be up and schedule # application notification call if it was down or # returned a different probe result cage_info = self._up_cages.setdefault(cage, {}).setdefault(node, {}) if not cage_info or cage_info["probe_result"] != probe_result: self._schedule_up_down_event(node, cage, "up", probe_result) cage_info.update(location=location, probe_result=probe_result) elif up_down == "down": # remove the cage from cages known to be up and schedule # application notification call it was up if self._up_cages.setdefault(cage, {}).pop(node, None): self._schedule_up_down_event(node, cage, "down") except: pmnc.log.error(exc_string()) # log and ignore if current_thread().stopped(): return False return True ################################### # this method is called by the _probe_thread in response to change # of some cage's state detected in _poll_up_down_queue def _schedule_up_down_event(self, node, cage, up_down, probe_result=None): # application notification invokes methods from health_monitor_event module # and must be executed just like a regular request from some interface request = pmnc.interfaces.begin_request( timeout=self._request_timeout, interface="__health_monitor__", protocol="n/a", parameters=dict(auth_tokens=dict()), description="cage {0:s}.{1:s} is {2:s}".format(node, cage, up_down), ) # note that this request is not waited upon pmnc.interfaces.enqueue(request, self.wu_process_event, (node, cage, up_down, probe_result)) ################################### # this method is invoked by one of the interfaces pool threads to register # the event of some cage going up or down by calling an appropriate method # from the health_monitor_event module @typecheck def wu_process_event(self, node: str, cage: str, up_down: one_of("up", "down"), probe_result: optional(dict)): try: # see for how long the request was on the execution queue up to this moment # and whether it has expired in the meantime, if it did there is no reason # to proceed and we simply bail out if pmnc.request.expired: pmnc.log.error("request has expired and will not be processed") success = False return # goes through finally section below with pmnc.performance.request_processing(): self._process_event(node, cage, up_down, probe_result) except: pmnc.log.error(exc_string()) # log and ignore success = False else: success = True finally: # the request ends itself pmnc.interfaces.end_request(success) # possibly way after deadline ################################### def _process_event(self, node, cage, up_down, probe_result): if up_down == "up": pmnc.health_monitor_event.cage_up(node, cage, probe_result) elif up_down == "down": pmnc.health_monitor_event.cage_down(node, cage)
class HealthMonitor: def __init__(self, **kwargs): self._rpc_interface = pmnc.interfaces.get_interface("rpc") if self._rpc_interface is None: raise Exception("health monitor requires enabled rpc interface") self._probe_thread_pool = pmnc.shared_pools.get_private_thread_pool() self._up_cages = {} # { cage: { node: { location: ..., probe_result: ... } } } self._up_down_queue = InterlockedQueue() self._request_timeout = pmnc.config_interfaces.get("request_timeout") # this is now static if pmnc.request.self_test == __name__: # self-test self._process_event = kwargs["process_event"] self._probe_cage = kwargs["probe_cage"] ################################### def start(self): self._probe_thread = HeavyThread(target = self._probe_thread_proc, name = "health_monitor:probe") # always called "health_monitor" self._probe_thread.start() def stop(self): self._probe_thread.stop() ################################### # this method is executed in a private thread and is scheduling probe calls # to cages known to the RPC interface or previously probed and found to be up def _probe_thread_proc(self): per_cage_interval = 0.0 # calls to _poll_up_down_queue are interleaved and allow this thread # to maintain structures such as _up_cages in response to events # posted by the probe threads to the _up_down_queue while self._poll_up_down_queue(per_cage_interval): try: # extract all cages currently known to the rpc interface and # merge them with cages previously probed and found to be up, # except for the health_monitor cage itself should be skipped probe_cages = \ { known_cage: { known_node: dict(location = known_location, probe_result = None) for known_node, known_location in self._rpc_interface.get_nodes(known_cage).items() } for known_cage in self._rpc_interface.get_cages() if known_cage != "health_monitor" } self._merge_cages(probe_cages, self._up_cages) probe_period = pmnc.config.get("probe_period") per_cage_interval = probe_period / (len(probe_cages) + 1) # walk through all cages to be probed and schedule calls to probe # to a private thread pool using fake unregistered requests for cage, nodes in probe_cages.items(): for node, cage_info in nodes.items(): cage_location = cage_info["location"] # note that the requests created here are not registered with # interfaces and enqueued to a different pool too, they are # therefore entitled to termination without warning at shutdown, # this is ok, because they do no useful work for the clients request = Request(timeout = self._request_timeout, interface = "__health_monitor__", protocol = "n/a", parameters = dict(auth_tokens = dict()), description = "probing cage {0:s} at {1:s}".format(cage, cage_location)) self._probe_thread_pool.enqueue(request, self.wu_probe_cage, (node, cage, cage_location, cage_info["probe_result"]), {}) # then again yield to polling the queue for a while if not self._poll_up_down_queue(per_cage_interval): break except: pmnc.log.error(exc_string()) # log and ignore ################################### # this method merges cages known to the RPC interface with cages # previously probed and known to be up, such merging is necessary # because if a cage dies just before its next advertisement broadcast, # it would disappear from known, but will not be probed again and # hence thought to be up forever @staticmethod def _merge_cages(known_cages: dict, up_cages: dict): probe_cages = known_cages # merging in place for up_cage, up_nodes in up_cages.items(): for up_node, up_cage_info in up_nodes.items(): probe_nodes = probe_cages.setdefault(up_cage, {}) if up_node in probe_nodes: cage_info = probe_nodes[up_node] if cage_info["location"] == up_cage_info["location"]: cage_info.update(probe_result = up_cage_info["probe_result"]) else: cage_info.update(probe_result = "restarted") # note this case else: probe_nodes[up_node] = up_cage_info ################################### # a call to this method is enqueued to a private thread pool # for each cage to probe on every pass of _probe_thread def wu_probe_cage(self, node, cage, location, prev_probe_result): if pmnc.request.expired: # no need to report anything for a probing request return if pmnc.log.debug: pmnc.log.debug("sending probe") try: probe_result = self._probe_cage(node, cage, location) except: pmnc.log.warning("probe failed: {0:s}".format(exc_string())) self._up_down_queue.push((node, cage, "down")) else: if pmnc.log.debug: pmnc.log.debug("probe returned successfully") if prev_probe_result == "restarted": # if the cage has restarted self._up_down_queue.push((node, cage, "down")) # we push "down" event first self._up_down_queue.push((node, cage, "up", location, probe_result)) ################################### # this method is invoked by one of the private pool threads # to send the actual probe call to the cage being probed @typecheck def _probe_cage(self, node, cage, location) -> dict: # health monitor has to create rpc resources manually, not using # pmnc(cage) syntax, because we need to access exact cage at exact # node and location (i.e. host and port) and to avoid discovery connect_timeout = pmnc.config_resource_rpc.get("discovery_timeout") rpc = pmnc.protocol_rpc.Resource("{0:s}.{1:s}".format(node, cage), broadcast_address = ("n/a", 0), discovery_timeout = connect_timeout, multiple_timeout_allowance = 0.0, flock_id = "unused", exact_locations = { cage: location }, # this prevents discovery pool__resource_name = cage) rpc.connect() try: rpc.begin_transaction("", source_module_name = __name__, transaction_options = {}, resource_args = (), resource_kwargs = {}) try: probe_result = rpc.health_monitor_event.probe() # there, an RPC call except: rpc.rollback() raise else: rpc.commit() finally: rpc.disconnect() return probe_result # if the cage returns anything but a dict, it is considered a failure ################################### # this method is called by the _probe_thread during its idle times # to fetch up/down events posted to the _up_down_queue by the probe # threads and in response to maintain structures such as _up_cages def _poll_up_down_queue(self, timeout: float) -> bool: # returns "should keep running" poll_timeout = Timeout(timeout) while not poll_timeout.expired: pop_timeout = Timeout(min(poll_timeout.remain, 1.0)) while not pop_timeout.expired: event = pop_timeout.pop(self._up_down_queue) if event is not None: try: node, cage, up_down, *args = event if up_down == "up": location, probe_result = args # add the cage to cages known to be up and schedule # application notification call if it was down or # returned a different probe result cage_info = self._up_cages.setdefault(cage, {}).setdefault(node, {}) if not cage_info or cage_info["probe_result"] != probe_result: self._schedule_up_down_event(node, cage, "up", probe_result) cage_info.update(location = location, probe_result = probe_result) elif up_down == "down": # remove the cage from cages known to be up and schedule # application notification call it was up if self._up_cages.setdefault(cage, {}).pop(node, None): self._schedule_up_down_event(node, cage, "down") except: pmnc.log.error(exc_string()) # log and ignore if current_thread().stopped(): return False return True ################################### # this method is called by the _probe_thread in response to change # of some cage's state detected in _poll_up_down_queue def _schedule_up_down_event(self, node, cage, up_down, probe_result = None): # application notification invokes methods from health_monitor_event module # and must be executed just like a regular request from some interface request = pmnc.interfaces.begin_request( timeout = self._request_timeout, interface = "__health_monitor__", protocol = "n/a", parameters = dict(auth_tokens = dict()), description = "cage {0:s}.{1:s} is {2:s}".format(node, cage, up_down)) # note that this request is not waited upon pmnc.interfaces.enqueue(request, self.wu_process_event, (node, cage, up_down, probe_result)) ################################### # this method is invoked by one of the interfaces pool threads to register # the event of some cage going up or down by calling an appropriate method # from the health_monitor_event module @typecheck def wu_process_event(self, node: str, cage: str, up_down: one_of("up", "down"), probe_result: optional(dict)): try: # see for how long the request was on the execution queue up to this moment # and whether it has expired in the meantime, if it did there is no reason # to proceed and we simply bail out if pmnc.request.expired: pmnc.log.error("request has expired and will not be processed") success = False return # goes through finally section below with pmnc.performance.request_processing(): self._process_event(node, cage, up_down, probe_result) except: pmnc.log.error(exc_string()) # log and ignore success = False else: success = True finally: # the request ends itself pmnc.interfaces.end_request(success) # possibly way after deadline ################################### def _process_event(self, node, cage, up_down, probe_result): if up_down == "up": pmnc.health_monitor_event.cage_up(node, cage, probe_result) elif up_down == "down": pmnc.health_monitor_event.cage_down(node, cage)
class _SMPPConnection: @typecheck def __init__(self, name: str, in_q: with_attr("push"), out_q: with_attr("pop"), inflight: with_attr("add", "remove"), *, server_address: (str, int), connect_timeout: float, response_timeout: float, system_id: str, password: str, system_type: str, esme_ton: byte, esme_npi: byte, esme_addr: str, bind_pdu: one_of(BindReceiverPDU, BindTransmitterPDU, BindTransceiverPDU)): self._name = name # input and output queues and inflight requests registry are external # to the connection object and remain persistent across disconnects to # prevent losing packets self._in_q = in_q self._out_q = out_q self._inflight = inflight self._server_address = server_address self._connect_timeout = connect_timeout self._response_timeout = response_timeout self._bind_pdu = \ bind_pdu.create(system_id = system_id.encode("ascii", "replace"), password = password.encode("ascii", "replace"), system_type = system_type.encode("ascii", "replace"), interface_version = 0x34, addr_ton = esme_ton, addr_npi = esme_npi, address_range = esme_addr.encode("ascii", "replace")) self._bound = Event() self._failed = Event() failed = property(lambda self: self._failed.is_set()) ################################### def start(self): self._bind_timeout = Timeout(self._connect_timeout) self._connect(self._bind_timeout.remain) self._start_threads() try: self._wait_for_bind() except: self.stop() raise def stop(self): self._stop_threads() self._disconnect() ################################### def _connect(self, timeout): s = socket(AF_INET, SOCK_STREAM) try: s.settimeout(timeout or 0.01) s.connect(self._server_address) s.setblocking(False) except: s.close() raise else: self._socket = s def _disconnect(self): try: self._socket.close() except: pmnc.log(exc_string()) # log and ignore ################################### def _start_threads(self): self._reader = HeavyThread(target = self._reader_proc, name = "{0:s}/rd".format(self._name)) self._reader.start() self._writer = HeavyThread(target = self._writer_proc, name = "{0:s}/wr".format(self._name)) self._writer.start() def _stop_threads(self): self._writer.stop() # the writer thread must be stopped first, because it depends self._reader.stop() # on the reader thread to be active to perform unbind ################################### def _wait_for_bind(self): while not self._bind_timeout.expired: self._bound.wait(min(0.5, self._bind_timeout.remain)) # resort to polling because two if self._bound.is_set(): # events need to be waited upon break if self._failed.is_set(): raise Exception("binding attempt failed") else: raise Exception("binding attempt timed out") ################################### class ThreadStopped(Exception): pass ################################### def read(self, n): while len(self._read_data) < n: ss = select([self._socket], [], [], 1.0)[0] if current_thread().stopped(): # this particular exception is accounted for raise _SMPPConnection.ThreadStopped("connection is being closed") # and masked out in _reader_proc below if not ss: continue self._read_data += ss[0].recv(4096) result, self._read_data = self._read_data[:n], self._read_data[n:] return result ################################### def _reader_proc(self): self._read_data = b"" while True: try: try: pdu = PDU.read(self) except SMPPPDUReadError as e: # mask out the exception from if isinstance(e.__cause__, _SMPPConnection.ThreadStopped): # the thread being stopped break # while else: raise if isinstance(pdu, RequestPDU): if pmnc.log.debug: pmnc.log.debug("<< [REQ] {0:s}".format(str(pdu))) self._in_q.push(pdu) elif isinstance(pdu, ResponsePDU): req = self._inflight.remove(pdu) if req is not None: if pmnc.log.debug: pmnc.log.debug("<< [RSP] {0:s}".format(str(pdu))) req.set_response(pdu) else: pmnc.log.warning("<< [RSP?!] {0:s}".format(str(pdu))) else: pmnc.log.warning("<< [???] {0:s}".format(str(pdu))) except: pmnc.log.error(exc_string()) self._failed.set() break ################################### def _writer_proc(self): try: # perform synchronous bind, using cumulative timeout if not self._write_pdu(self._bind_pdu, False, self._bind_timeout): return _wait_response(self._bind_pdu, self._bind_timeout.remain) self._bound.set() while not current_thread().stopped(): # lifetime loop pdu = self._out_q.pop(1.0) if pdu and not self._write_pdu(pdu, False, Timeout(self._response_timeout)): break # perform synchronous unbind unbind_pdu = UnbindPDU.create() unbind_timeout = Timeout(self._response_timeout) self._write_pdu(unbind_pdu, True, unbind_timeout) _wait_response(unbind_pdu, unbind_timeout.remain) except: pmnc.log.error(exc_string()) self._failed.set() ################################### def _write_pdu(self, pdu, stopped, timeout): request_pdu = isinstance(pdu, RequestPDU) if request_pdu: self._inflight.add(pdu) try: self._write_packet(pdu.serialize(), stopped, timeout) except _SMPPConnection.ThreadStopped: if request_pdu: self._inflight.remove(pdu) return False except: if request_pdu: self._inflight.remove(pdu) raise if pmnc.log.debug: if request_pdu: pmnc.log.debug(">> [REQ] {0:s}".format(str(pdu))) elif isinstance(pdu, ResponsePDU): pmnc.log.debug(">> [RSP] {0:s}".format(str(pdu))) else: pmnc.log.debug(">> [???] {0:s}".format(str(pdu))) return True ################################### def _write_packet(self, packet, stopped, timeout): while packet: ss = select([], [self._socket], [], min(timeout.remain, 1.0))[1] if current_thread().stopped() and not stopped: raise _SMPPConnection.ThreadStopped("connection is being closed") if timeout.expired: raise Exception("timeout writing packet") if ss: sent = ss[0].send(packet) packet = packet[sent:]
class Interface: # SMPP interface @typecheck def __init__(self, name: str, *, server_address: (str, int), connect_timeout: float, response_timeout: float, ping_interval: optional(float), system_id: str, password: str, system_type: str, esme_ton: byte, esme_npi: byte, esme_addr: str, esme_type: one_of("rcvr", "xmit", "xcvr"), request_timeout: optional(float) = None, **kwargs): # this kwargs allows for extra application-specific # settings in config_interface_smpp_X.py self._name = name self._response_timeout = response_timeout if ping_interval: self._ping_timeout = Timeout(ping_interval) self._ping_response_timeout = Timeout(response_timeout) else: self._ping_timeout = self._ping_response_timeout = None self._ping_request = None self._in_q = InterlockedQueue() self._out_q = InterlockedQueue() self._inflight = InflightRequests() self._ceased = Event() if esme_type == "rcvr": bind_pdu = BindReceiverPDU elif esme_type == "xmit": bind_pdu = BindTransmitterPDU elif esme_type == "xcvr": bind_pdu = BindTransceiverPDU self._create_connection = \ lambda: _SMPPConnection(name, self._in_q, self._out_q, self._inflight, server_address = server_address, connect_timeout = connect_timeout, response_timeout = response_timeout, system_id = system_id, password = password, system_type = system_type, esme_ton = esme_ton, esme_npi = esme_npi, esme_addr = esme_addr, bind_pdu = bind_pdu) self._request_timeout = request_timeout or \ pmnc.config_interfaces.get("request_timeout") # this is now static if pmnc.request.self_test == __name__: # self-test self._process_request = kwargs["process_request"] name = property(lambda self: self._name) ceased = property(lambda self: self._ceased.is_set()) ################################### def start(self): self._maintainer = HeavyThread(target = self._maintainer_proc, name = "{0:s}/mnt".format(self.name)) self._maintainer.start() def cease(self): self._ceased.set() self._maintainer.stop() def stop(self): pass ################################### def _maintainer_proc(self): while not current_thread().stopped(): try: # try to establish a connection, do it infinitely or until the interface is stopped while True: try: self._connection = self._create_connection() self._connection.start() except: pmnc.log.error(exc_string()) failure_timeout = max(self._request_timeout, 30.0) if current_thread().stopped(failure_timeout): return else: break # while True try: while not current_thread().stopped() and not self._connection.failed: # process incoming PDUs req = self._in_q.pop(1.0) if req is not None: self._handle_pdu(req) # if there is an outstanding ping request, check for response if self._ping_request and self._ping_response_timeout.expired: ping_request, self._ping_request = self._ping_request, None _wait_response(ping_request, 0.001) # if it's time to send another ping request, do so if self._ping_timeout and self._ping_timeout.expired: try: self._ping_request = EnquireLinkPDU.create() self._out_q.push(self._ping_request) self._ping_response_timeout.reset() finally: self._ping_timeout.reset() finally: self._connection.stop() except: pmnc.log.error(exc_string()) # log and ignore ################################### # this method processes the request PDUs received by this interface from SMSC def _handle_pdu(self, req): if isinstance(req, EnquireLinkPDU): # respond to pings automatically resp = req.create_response() self._out_q.push(resp) else: # note that this interface does not wait for its requests to complete request = pmnc.interfaces.begin_request( timeout = self._request_timeout, interface = self._name, protocol = "smpp", parameters = dict(auth_tokens = dict()), description = "incoming {0:s}".format(req)) pmnc.interfaces.enqueue(request, self.wu_handle_pdu, (req, ), {}) ################################### @typecheck def wu_handle_pdu(self, req: RequestPDU): try: # see for how long the request was on the execution queue up to this moment # and whether it has expired in the meantime, if it did there is no reason # to proceed and we simply bail out if pmnc.request.expired: pmnc.log.error("request has expired and will not be processed") success = False return # goes through finally section below with pmnc.performance.request_processing(): request = dict(pdu = req) response = dict(pdu = req.create_nack(error_codes.ESME_RUNKNOWNERR)) try: self._process_request(request, response) except: response["pdu"] = req.create_nack(error_codes.ESME_RSYSERR) raise finally: self._out_q.push(response["pdu"]) except: pmnc.log.error(exc_string()) # log and ignore success = False else: success = True finally: # the request ends itself pmnc.interfaces.end_request(success) # possibly way after deadline ################################### def _process_request(self, request, response): handler_module_name = "interface_{0:s}".format(self._name) pmnc.__getattr__(handler_module_name).process_request(request, response) ################################### # this method is called by the coupled resources to send a request PDU to SMSC @typecheck def send(self, req: RequestPDU, timeout: optional(float) = None) -> optional(ResponsePDU): self._out_q.push(req) if timeout is not None: return _wait_response(req, min(timeout, self._response_timeout))
class Interface(AdapterHost): # JMS interface @typecheck def __init__(self, name: str, *, java: os_path.isfile, arguments: tuple_of(str), classpath: str, jndi: dict_of(str, str), factory: str, queue: str, username: str, password: str, request_timeout: optional(float) = None, **kwargs): # this kwargs allows for extra application-specific # settings in config_interface_jms_X.py self._name = name AdapterHost.__init__(self, "org.pythomnic.jms.Receiver", java = java, arguments = arguments, classpath = classpath, jndi = jndi, factory = factory, queue = queue, username = username, password = password) self._request_timeout = request_timeout or \ pmnc.config_interfaces.get("request_timeout") # this is now static if pmnc.request.self_test == __name__: # self-test self._process_request = kwargs["process_request"] name = property(lambda self: self._name) ################################### def start(self): self._maintainer = HeavyThread(target = self._maintainer_proc, name = "{0:s}:mnt".format(self._name)) self._maintainer.start() def cease(self): self._maintainer.stop() def stop(self): pass ################################### # this thread keeps the interface up by (re)starting # the adapter and its controlling i/o threads def _maintainer_proc(self): while not current_thread().stopped(): while True: # keep trying to start the adapter try: self._start_adapter("interface", self._name, Timeout(15.0)) except: pmnc.log.error(exc_string()) failure_timeout = max(self._request_timeout, 30.0) if current_thread().stopped(failure_timeout): return else: break # now that the adapter is running, keep receiving messages # until the adapter exits or the interface is ceased try: try: while self._adapter_running(): # even when the queue is idle, the adapter should be sending in a ping once in 3 seconds receive_timeout = Timeout(self._request_timeout + 3.0) pkt = receive_timeout.pop(self._stdout_queue) if pkt is None: raise Exception("adapter process failed to produce a message") if "XPmncError" in pkt: # adapter reports error, abort raise Exception(pkt["XPmncError"]) request_id = pkt.pop("XPmncRequestID") if current_thread().stopped(): # any command after shutdown gets an EXIT response response = Packet(XPmncResponse = "EXIT", XPmncRequestID = request_id) self._stdin_queue.push(response) break request = pkt.pop("XPmncRequest") if request == "NOOP": # ping always gets an OK response response = Packet(XPmncResponse = "OK", XPmncRequestID = request_id) self._stdin_queue.push(response) elif request == "RECEIVE": # process an incoming message message_id = pkt["JMSMessageID"] if message_id not in self._processed_messages: # don't process the message again success = self._process_message(message_id, pkt, receive_timeout.remain) else: success = True response = Packet(XPmncResponse = success and "COMMIT" or "ROLLBACK", XPmncRequestID = request_id, XPmncMessageID = message_id) self._stdin_queue.push(response) else: raise Exception("invalid request") finally: self._stop_adapter(Timeout(5.0)) # make sure the adapter is not running except: pmnc.log.error(exc_string()) # log and ignore ################################### # request processing in a message-oriented interface is rather straightforward def _process_message(self, message_id, pkt, request_timeout): message_text = pkt.pop("XPmncMessageText") message_description = "JMS message {0:s}{1:s}".format(message_id, ", corr.id {0:s}".format(pkt["JMSCorrelationID"]) if pkt.get("JMSCorrelationID") else "") request = pmnc.interfaces.begin_request( timeout = min(request_timeout, self._request_timeout), interface = self._name, protocol = "jms", parameters = dict(auth_tokens = dict()), description = message_description) try: pmnc.interfaces.enqueue(request, self.wu_process_request, (message_id, message_text, pkt.copy())).wait() except WorkUnitTimedOut: pmnc.log.error("{0:s} processing timed out".format(message_description)) success = None except: pmnc.log.error("{0:s} processing failed: {1:s}".\ format(message_description, exc_string())) success = False else: if pmnc.log.debug: pmnc.log.debug("{0:s} processing succeeded".format(message_description)) self._processed_messages.add(message_id) success = True finally: pmnc.interfaces.end_request(success, request) return success == True ################################### # this method is a work unit executed by one of the interface pool threads # if this method fails, the exception is rethrown in _process_message in wait() @typecheck def wu_process_request(self, message_id: str, message_text: str, headers: dict): # see for how long the request was on the execution queue up to this moment # and whether it has expired in the meantime, if it did there is no reason # to proceed and we simply bail out if pmnc.request.expired: pmnc.log.error("request has expired and will not be processed") return try: with pmnc.performance.request_processing(): request = dict(message_id = message_id, message_text = message_text, headers = headers) self._process_request(request, {}) except: pmnc.log.error(exc_string()) # don't allow an exception to be silenced raise # when this work unit is not waited upon ################################### def _process_request(self, request, response): handler_module_name = "interface_{0:s}".format(self._name) pmnc.__getattr__(handler_module_name).process_request(request, response)
class Interface: # file-loading interface @typecheck def __init__(self, name: str, *, source_directory: os_path.isdir, filename_regex: str, interval: float, request_timeout: optional(float) = None, **kwargs): # this kwargs allows for extra application-specific # settings in config_interface_file_X.py self._name = name self._source_directory = source_directory self._valid_filename = by_regex("^{0:s}$".format(filename_regex)) self._interval = interval self._request_timeout = request_timeout or \ pmnc.config_interfaces.get("request_timeout") # this is now static # this set contains names of files that have been processed # but not deleted due to filesystem deletion failure self._processed_files = set() if pmnc.request.self_test == __name__: # self-test self._process_request = kwargs["process_request"] name = property(lambda self: self._name) ################################### def start(self): self._spooler = HeavyThread(target = self._spooler_proc, name = "{0:s}:spool".format(self._name)) self._spooler.start() def cease(self): self._spooler.stop() def stop(self): pass ################################### # this thread periodically scans the configured directory for files def _spooler_proc(self): interval = self._interval while not current_thread().stopped(interval): try: # enumerate matching files in the source directory file_names = [ os_path.join(self._source_directory, file_name) for file_name in os.listdir(self._source_directory) if self._valid_filename(file_name) and os_path.isfile(os_path.join(self._source_directory, file_name)) ] if not file_names: interval = self._interval # no files, rescan after a delay continue # sort and process the file names in alphabetical order file_names.sort() for file_name in file_names: if current_thread().stopped(): # processing of all files may take a long time break # for elif file_name in self._processed_files: # just remove the already processed file self._remove_file(file_name) elif self._process_file(file_name): # remove the file if it has been processed self._remove_file(file_name) else: interval = self._interval # processing of some file failed, break # for # rescan after a delay else: # all the files have been processed, rescan immediately, interval = 0.0 # hoping for the new files to appear in the meantime except: pmnc.log.error(exc_string()) interval = self._interval ################################### def _remove_file(self, file_name): try: _retry_remove_file(file_name, 2.0) except: pmnc.log.warning(exc_string()) else: self._processed_files.remove(file_name) ################################### def _process_file(self, file_name): request = pmnc.interfaces.begin_request( timeout = self._request_timeout, interface = self._name, protocol = "file", parameters = dict(auth_tokens = dict()), description = "file {0:s}".format(file_name)) # enqueue the request and wait for its completion try: pmnc.interfaces.enqueue(request, self.wu_process_request, (file_name, )).wait() except WorkUnitTimedOut: pmnc.log.error("file processing timed out") success = None except: pmnc.log.error("file processing failed: {0:s}".format(exc_string())) success = False else: if pmnc.log.debug: pmnc.log.debug("file processing succeeded") self._processed_files.add(file_name) success = True finally: pmnc.interfaces.end_request(success, request) return success == True ################################### # this method is a work unit executed by one of the interface pool threads # if this method fails, the exception is rethrown in _process_file in wait() @typecheck def wu_process_request(self, file_name: str): # see for how long the request was on the execution queue up to this moment # and whether it has expired in the meantime, if it did there is no reason # to proceed and we simply bail out if pmnc.request.expired: pmnc.log.error("request has expired and will not be processed") return try: with pmnc.performance.request_processing(): request = dict(file_name = os_path.join(self._source_directory, file_name)) self._process_request(request, {}) except: pmnc.log.error(exc_string()) # don't allow an exception to be silenced raise # when this work unit is not waited upon ################################### def _process_request(self, request, response): handler_module_name = "interface_{0:s}".format(self._name) pmnc.__getattr__(handler_module_name).process_request(request, response)
class Interface: @typecheck def __init__(self, name: str, *, listener_address: (str, int), request_timeout: optional(float) = None, **kwargs): # this kwargs allows for extra application-specific # settings in config_interface_udp_X.py self._name = name self._listener_address = listener_address self._request_timeout = request_timeout or \ pmnc.config_interfaces.get("request_timeout") # this is now static if pmnc.request.self_test == __name__: # self-test self._process_request = kwargs["process_request"] name = property(lambda self: self._name) listener_address = property(lambda self: self._listener_address) ################################### def start(self): self._listener = HeavyThread(target = self._listener_proc, name = "{0:s}:lsn".format(self._name)) self._listener.start() ################################### def cease(self): self._listener.stop() ################################### def stop(self): pass ################################### def _create_server_socket(self) -> socket: s = socket(AF_INET, SOCK_DGRAM) try: s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) if have_reuse_port: s.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1) s.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) s.bind(self._listener_address) except: s.close() raise else: return s ################################### def _listener_proc(self): while not current_thread().stopped(): try: s = self._create_server_socket() try: pmnc.log.message("started listening for packets at {0[0]:s}:{0[1]:d}".\ format(self._listener_address)) while not current_thread().stopped(): if not select([s], [], [], 1.0)[0]: continue try: packet_b, (client_addr, client_port) = s.recvfrom(57344) except socket_timeout: continue except socket_error as e: if e.args[0] == 10054: # workaround for this issue: http://support.microsoft.com/kb/263823 continue else: raise if pmnc.log.debug: pmnc.log.debug("incoming UDP packet from {0:s}:{1:d}, {2:d} byte(s)".\ format(client_addr, client_port, len(packet_b))) # the listener thread initiates the received packet processing try: self._enqueue_packet(packet_b, client_addr, client_port) except: pmnc.log.error(exc_string()) # log and ignore finally: s.close() pmnc.log.message("stopped listening") except: pmnc.log.error(exc_string()) # log and ignore ################################### def _enqueue_packet(self, packet_b, client_addr, client_port): # create a new request for processing the message request = pmnc.interfaces.begin_request( timeout = self._request_timeout, interface = self._name, protocol = "udp", parameters = dict(auth_tokens = dict(peer_ip = client_addr)), description = "UDP packet from {0:s}, {1:d} byte(s)".\ format(client_addr, len(packet_b))) # enqueue the request but do not wait for its completion pmnc.interfaces.enqueue(request, self.wu_process_request, (packet_b, )) ################################### def wu_process_request(self, packet_b): try: # see for how long the request was on the execution queue up to this moment # and whether it has expired in the meantime, if it did there is no reason # to proceed and we simply bail out if pmnc.request.expired: pmnc.log.error("request has expired and will not be processed") success = False return with pmnc.performance.request_processing(): request = dict(packet = packet_b) self._process_request(request, {}) except: pmnc.log.error(exc_string()) success = False else: success = True finally: # the request ends itself pmnc.interfaces.end_request(success) # possibly way after deadline ################################### def _process_request(self, request, response): handler_module_name = "interface_{0:s}".format(self._name) pmnc.__getattr__(handler_module_name).process_request(request, response)