class InflightRequests: _request_ttl = timedelta(minutes = 10) def __init__(self): self._lock = Lock() self._pdus = {} self._cleanup_timeout = Timeout(60.0) def _cleanup(self): now = datetime.now() if self._cleanup_timeout.expired: try: self._pdus = { k: pdu_exp for k, pdu_exp in self._pdus.items() if now < pdu_exp[1] } finally: self._cleanup_timeout.reset() return now @typecheck def add(self, pdu: RequestPDU): with self._lock: now = self._cleanup() self._pdus[pdu.sequence_number] = (pdu, now + self._request_ttl) @typecheck def remove(self, pdu: PDU) -> optional(RequestPDU): with self._lock: self._cleanup() pdu_exp = self._pdus.pop(pdu.sequence_number, None) if pdu_exp: return pdu_exp[0]
class PooledThread(Resource): def __init__(self, name, release): Resource.__init__(self, name) self._release = release self._ready, self._queue = Event(), InterlockedQueue() if __name__ == "__main__": self._timeout = Timeout(3.0) else: self._timeout = Timeout(60.0) self._count = 0 def _expired(self): return self._timeout.expired or Resource._expired(self) def connect(self): Resource.connect(self) self._thread = LightThread(target=self._thread_proc, name="{0:s}:?".format(self.name)) self._thread.start() self._ready.wait( 3.0) # this may spend waiting slightly less, but it's ok if not self._ready.is_set(): self._queue.push( exit) # just in case the thread has in fact started raise Exception("new thread failed to start in 3.0 seconds") def _thread_proc(self): self._ready.set() while True: # exits upon processing of exit pushed in disconnect() try: self._count += 1 thread_name = "{0:s}:{1:d}".format(self.name, self._count) current_thread().name = thread_name work_unit = self._queue.pop() work_unit() self._timeout.reset() finally: self._release( self) # this actually invokes ThreadPool._release # this method may be called by external thread (ex. pool sweep) # or by this thread itself, and posts an exit kind of work unit def disconnect(self): try: if current_thread() is not self._thread: self._queue.push(exit) self._thread.join(3.0) finally: Resource.disconnect(self) # this method is called by the thread pool to post a work unit # to this thread, as well as by the thread itself at disconnect def push(self, work_unit): self._queue.push(work_unit)
class PooledThread(Resource): def __init__(self, name, release): Resource.__init__(self, name) self._release = release self._ready, self._queue = Event(), InterlockedQueue() if __name__ == "__main__": self._timeout = Timeout(3.0) else: self._timeout = Timeout(60.0) self._count = 0 def _expired(self): return self._timeout.expired or Resource._expired(self) def connect(self): Resource.connect(self) self._thread = LightThread(target = self._thread_proc, name = "{0:s}:?".format(self.name)) self._thread.start() self._ready.wait(3.0) # this may spend waiting slightly less, but it's ok if not self._ready.is_set(): self._queue.push(exit) # just in case the thread has in fact started raise Exception("new thread failed to start in 3.0 seconds") def _thread_proc(self): self._ready.set() while True: # exits upon processing of exit pushed in disconnect() try: self._count += 1 thread_name = "{0:s}:{1:d}".format(self.name, self._count) current_thread().name = thread_name work_unit = self._queue.pop() work_unit() self._timeout.reset() finally: self._release(self) # this actually invokes ThreadPool._release # this method may be called by external thread (ex. pool sweep) # or by this thread itself, and posts an exit kind of work unit def disconnect(self): try: if current_thread() is not self._thread: self._queue.push(exit) self._thread.join(3.0) finally: Resource.disconnect(self) # this method is called by the thread pool to post a work unit # to this thread, as well as by the thread itself at disconnect def push(self, work_unit): self._queue.push(work_unit)
class _WorkSource: @typecheck def __init__(self, idle_timeout: float): self._idle_timeout = Timeout(idle_timeout) self._has_work = False self._working = False def add_work(self): self._has_work = True def begin_work(self): if not self._working and (self._has_work or self._idle_timeout.expired): self._has_work = False self._working = True return True else: return False def end_work(self): assert self._working self._working = False self._idle_timeout.reset()
class ModuleLocator: @typecheck def __init__(self, cage_directory: os_path.isdir, cache_timeout: float, settle_timeout: float): self._cage_directory = os_path.normpath(cage_directory) shared_directory = os_path.normpath( os_path.join(cage_directory, "..", ".shared")) self._shared_directory = os_path.isdir( shared_directory) and shared_directory or None self._timeout = Timeout(cache_timeout) self._settle_timeout_sec = settle_timeout self._modules = self._settle_modules = self._settle_timeout = None self._lock = Lock() ################################### @staticmethod def _listdir(s): try: return listdir(s) except: return [] ################################### def _read_modules(self): modules = { module_name: os_path.join(self._shared_directory, module_name) for module_name in self._listdir(self._shared_directory) } modules.update({ module_name: os_path.join(self._cage_directory, module_name) for module_name in self._listdir(self._cage_directory) }) return modules ################################### def _get_modules(self): with self._lock: if self._modules is None: # initial state, the current directories contents is unknown modules = self._read_modules() self._modules = modules elif self._settle_timeout: # a change has been detected previously and is currently being settled if self._settle_timeout.expired: modules = self._read_modules() if modules != self._settle_modules: # another change occured since last time, keep settling self._settle_timeout.reset() self._settle_modules = modules else: # directories contents seems to have settled self._timeout.reset() self._modules = modules self._settle_modules = self._settle_timeout = None elif self._timeout.expired: # cached contents is refreshed self._timeout.reset() modules = self._read_modules() if modules != self._modules: # change detected, switch to settling if self._settle_timeout_sec > 0.0: self._settle_modules = modules self._settle_timeout = Timeout( self._settle_timeout_sec) else: self._modules = modules return self._modules ################################### @typecheck def locate(self, module_name: by_regex("^[A-Za-z0-9_-]{1,128}\\.pyc?$")): return self._get_modules().get(module_name)
class Interface: # SMPP interface @typecheck def __init__(self, name: str, *, server_address: (str, int), connect_timeout: float, response_timeout: float, ping_interval: optional(float), system_id: str, password: str, system_type: str, esme_ton: byte, esme_npi: byte, esme_addr: str, esme_type: one_of("rcvr", "xmit", "xcvr"), request_timeout: optional(float) = None, **kwargs): # this kwargs allows for extra application-specific # settings in config_interface_smpp_X.py self._name = name self._response_timeout = response_timeout if ping_interval: self._ping_timeout = Timeout(ping_interval) self._ping_response_timeout = Timeout(response_timeout) else: self._ping_timeout = self._ping_response_timeout = None self._ping_request = None self._in_q = InterlockedQueue() self._out_q = InterlockedQueue() self._inflight = InflightRequests() self._ceased = Event() if esme_type == "rcvr": bind_pdu = BindReceiverPDU elif esme_type == "xmit": bind_pdu = BindTransmitterPDU elif esme_type == "xcvr": bind_pdu = BindTransceiverPDU self._create_connection = \ lambda: _SMPPConnection(name, self._in_q, self._out_q, self._inflight, server_address = server_address, connect_timeout = connect_timeout, response_timeout = response_timeout, system_id = system_id, password = password, system_type = system_type, esme_ton = esme_ton, esme_npi = esme_npi, esme_addr = esme_addr, bind_pdu = bind_pdu) self._request_timeout = request_timeout or \ pmnc.config_interfaces.get("request_timeout") # this is now static if pmnc.request.self_test == __name__: # self-test self._process_request = kwargs["process_request"] name = property(lambda self: self._name) ceased = property(lambda self: self._ceased.is_set()) ################################### def start(self): self._maintainer = HeavyThread(target = self._maintainer_proc, name = "{0:s}/mnt".format(self.name)) self._maintainer.start() def cease(self): self._ceased.set() self._maintainer.stop() def stop(self): pass ################################### def _maintainer_proc(self): while not current_thread().stopped(): try: # try to establish a connection, do it infinitely or until the interface is stopped while True: try: self._connection = self._create_connection() self._connection.start() except: pmnc.log.error(exc_string()) failure_timeout = max(self._request_timeout, 30.0) if current_thread().stopped(failure_timeout): return else: break # while True try: while not current_thread().stopped() and not self._connection.failed: # process incoming PDUs req = self._in_q.pop(1.0) if req is not None: self._handle_pdu(req) # if there is an outstanding ping request, check for response if self._ping_request and self._ping_response_timeout.expired: ping_request, self._ping_request = self._ping_request, None _wait_response(ping_request, 0.001) # if it's time to send another ping request, do so if self._ping_timeout and self._ping_timeout.expired: try: self._ping_request = EnquireLinkPDU.create() self._out_q.push(self._ping_request) self._ping_response_timeout.reset() finally: self._ping_timeout.reset() finally: self._connection.stop() except: pmnc.log.error(exc_string()) # log and ignore ################################### # this method processes the request PDUs received by this interface from SMSC def _handle_pdu(self, req): if isinstance(req, EnquireLinkPDU): # respond to pings automatically resp = req.create_response() self._out_q.push(resp) else: # note that this interface does not wait for its requests to complete request = pmnc.interfaces.begin_request( timeout = self._request_timeout, interface = self._name, protocol = "smpp", parameters = dict(auth_tokens = dict()), description = "incoming {0:s}".format(req)) pmnc.interfaces.enqueue(request, self.wu_handle_pdu, (req, ), {}) ################################### @typecheck def wu_handle_pdu(self, req: RequestPDU): try: # see for how long the request was on the execution queue up to this moment # and whether it has expired in the meantime, if it did there is no reason # to proceed and we simply bail out if pmnc.request.expired: pmnc.log.error("request has expired and will not be processed") success = False return # goes through finally section below with pmnc.performance.request_processing(): request = dict(pdu = req) response = dict(pdu = req.create_nack(error_codes.ESME_RUNKNOWNERR)) try: self._process_request(request, response) except: response["pdu"] = req.create_nack(error_codes.ESME_RSYSERR) raise finally: self._out_q.push(response["pdu"]) except: pmnc.log.error(exc_string()) # log and ignore success = False else: success = True finally: # the request ends itself pmnc.interfaces.end_request(success) # possibly way after deadline ################################### def _process_request(self, request, response): handler_module_name = "interface_{0:s}".format(self._name) pmnc.__getattr__(handler_module_name).process_request(request, response) ################################### # this method is called by the coupled resources to send a request PDU to SMSC @typecheck def send(self, req: RequestPDU, timeout: optional(float) = None) -> optional(ResponsePDU): self._out_q.push(req) if timeout is not None: return _wait_response(req, min(timeout, self._response_timeout))