class InflightRequests:
_request_ttl = timedelta(minutes = 10)
def __init__(self):
self._lock = Lock()
self._pdus = {}
self._cleanup_timeout = Timeout(60.0)
def _cleanup(self):
now = datetime.now()
if self._cleanup_timeout.expired:
try:
self._pdus = { k: pdu_exp for k, pdu_exp in self._pdus.items() if now < pdu_exp[1] }
finally:
self._cleanup_timeout.reset()
return now
@typecheck
def add(self, pdu: RequestPDU):
with self._lock:
now = self._cleanup()
self._pdus[pdu.sequence_number] = (pdu, now + self._request_ttl)
@typecheck
def remove(self, pdu: PDU) -> optional(RequestPDU):
with self._lock:
self._cleanup()
pdu_exp = self._pdus.pop(pdu.sequence_number, None)
if pdu_exp: return pdu_exp[0]
def _writer_proc(self):
try:
# perform synchronous bind, using cumulative timeout
if not self._write_pdu(self._bind_pdu, False, self._bind_timeout):
return
_wait_response(self._bind_pdu, self._bind_timeout.remain)
self._bound.set()
while not current_thread().stopped(): # lifetime loop
pdu = self._out_q.pop(1.0)
if pdu and not self._write_pdu(pdu, False, Timeout(self._response_timeout)):
break
# perform synchronous unbind
unbind_pdu = UnbindPDU.create()
unbind_timeout = Timeout(self._response_timeout)
self._write_pdu(unbind_pdu, True, unbind_timeout)
_wait_response(unbind_pdu, unbind_timeout.remain)
except:
pmnc.log.error(exc_string())
self._failed.set()
def __init__(self, name, release):
Resource.__init__(self, name)
self._release = release
self._ready, self._queue = Event(), InterlockedQueue()
if __name__ == "__main__":
self._timeout = Timeout(3.0)
else:
self._timeout = Timeout(60.0)
self._count = 0
def __init__(self, name, release):
Resource.__init__(self, name)
self._release = release
self._ready, self._queue = Event(), InterlockedQueue()
if __name__ == "__main__":
self._timeout = Timeout(3.0)
else:
self._timeout = Timeout(60.0)
self._count = 0
class PooledThread(Resource):
def __init__(self, name, release):
Resource.__init__(self, name)
self._release = release
self._ready, self._queue = Event(), InterlockedQueue()
if __name__ == "__main__":
self._timeout = Timeout(3.0)
else:
self._timeout = Timeout(60.0)
self._count = 0
def _expired(self):
return self._timeout.expired or Resource._expired(self)
def connect(self):
Resource.connect(self)
self._thread = LightThread(target=self._thread_proc,
name="{0:s}:?".format(self.name))
self._thread.start()
self._ready.wait(
3.0) # this may spend waiting slightly less, but it's ok
if not self._ready.is_set():
self._queue.push(
exit) # just in case the thread has in fact started
raise Exception("new thread failed to start in 3.0 seconds")
def _thread_proc(self):
self._ready.set()
while True: # exits upon processing of exit pushed in disconnect()
try:
self._count += 1
thread_name = "{0:s}:{1:d}".format(self.name, self._count)
current_thread().name = thread_name
work_unit = self._queue.pop()
work_unit()
self._timeout.reset()
finally:
self._release(
self) # this actually invokes ThreadPool._release
# this method may be called by external thread (ex. pool sweep)
# or by this thread itself, and posts an exit kind of work unit
def disconnect(self):
try:
if current_thread() is not self._thread:
self._queue.push(exit)
self._thread.join(3.0)
finally:
Resource.disconnect(self)
# this method is called by the thread pool to post a work unit
# to this thread, as well as by the thread itself at disconnect
def push(self, work_unit):
self._queue.push(work_unit)
def __init__(self, name: str, *,
server_address: (str, int),
connect_timeout: float,
response_timeout: float,
ping_interval: optional(float),
system_id: str,
password: str,
system_type: str,
esme_ton: byte,
esme_npi: byte,
esme_addr: str,
esme_type: one_of("rcvr", "xmit", "xcvr"),
request_timeout: optional(float) = None,
**kwargs): # this kwargs allows for extra application-specific
# settings in config_interface_smpp_X.py
self._name = name
self._response_timeout = response_timeout
if ping_interval:
self._ping_timeout = Timeout(ping_interval)
self._ping_response_timeout = Timeout(response_timeout)
else:
self._ping_timeout = self._ping_response_timeout = None
self._ping_request = None
self._in_q = InterlockedQueue()
self._out_q = InterlockedQueue()
self._inflight = InflightRequests()
self._ceased = Event()
if esme_type == "rcvr":
bind_pdu = BindReceiverPDU
elif esme_type == "xmit":
bind_pdu = BindTransmitterPDU
elif esme_type == "xcvr":
bind_pdu = BindTransceiverPDU
self._create_connection = \
lambda: _SMPPConnection(name, self._in_q, self._out_q, self._inflight,
server_address = server_address,
connect_timeout = connect_timeout,
response_timeout = response_timeout,
system_id = system_id,
password = password,
system_type = system_type,
esme_ton = esme_ton,
esme_npi = esme_npi,
esme_addr = esme_addr,
bind_pdu = bind_pdu)
self._request_timeout = request_timeout or \
pmnc.config_interfaces.get("request_timeout") # this is now static
if pmnc.request.self_test == __name__: # self-test
self._process_request = kwargs["process_request"]
def __init__(self, cage_directory: os_path.isdir, cache_timeout: float,
settle_timeout: float):
self._cage_directory = os_path.normpath(cage_directory)
shared_directory = os_path.normpath(
os_path.join(cage_directory, "..", ".shared"))
self._shared_directory = os_path.isdir(
shared_directory) and shared_directory or None
self._timeout = Timeout(cache_timeout)
self._settle_timeout_sec = settle_timeout
self._modules = self._settle_modules = self._settle_timeout = None
self._lock = Lock()
class PooledThread(Resource):
def __init__(self, name, release):
Resource.__init__(self, name)
self._release = release
self._ready, self._queue = Event(), InterlockedQueue()
if __name__ == "__main__":
self._timeout = Timeout(3.0)
else:
self._timeout = Timeout(60.0)
self._count = 0
def _expired(self):
return self._timeout.expired or Resource._expired(self)
def connect(self):
Resource.connect(self)
self._thread = LightThread(target = self._thread_proc,
name = "{0:s}:?".format(self.name))
self._thread.start()
self._ready.wait(3.0) # this may spend waiting slightly less, but it's ok
if not self._ready.is_set():
self._queue.push(exit) # just in case the thread has in fact started
raise Exception("new thread failed to start in 3.0 seconds")
def _thread_proc(self):
self._ready.set()
while True: # exits upon processing of exit pushed in disconnect()
try:
self._count += 1
thread_name = "{0:s}:{1:d}".format(self.name, self._count)
current_thread().name = thread_name
work_unit = self._queue.pop()
work_unit()
self._timeout.reset()
finally:
self._release(self) # this actually invokes ThreadPool._release
# this method may be called by external thread (ex. pool sweep)
# or by this thread itself, and posts an exit kind of work unit
def disconnect(self):
try:
if current_thread() is not self._thread:
self._queue.push(exit)
self._thread.join(3.0)
finally:
Resource.disconnect(self)
# this method is called by the thread pool to post a work unit
# to this thread, as well as by the thread itself at disconnect
def push(self, work_unit):
self._queue.push(work_unit)
def test_WorkSources_end_work():
current_thread().stopped = lambda: False
wss = WorkSources(2, 30.0)
wss.add_work(1)
assert wss.begin_work(0.0) == (False, 1)
wss.add_work(0)
assert wss.begin_work(0.0) == (False, 0)
assert wss.begin_work(0.0) == (False, None)
wss.add_work(1)
assert wss.begin_work(0.0) == (False, None)
th_started = Event()
th_got_work = Event()
def th_proc():
th_started.set()
assert wss.begin_work(10.0) == (False, 1)
th_got_work.set()
th = HeavyThread(target=th_proc)
th.start()
th_started.wait()
t = Timeout(30.0)
wss.end_work(1)
th_got_work.wait()
assert t.remain > 29.0, t.remain
th.stop()
def test_performance():
t = Timeout(5.0)
c = 0
while not t.expired:
request = SubmitSmPDU.create(service_type = b"",
source_addr_ton = 0x00,
source_addr_npi = 0x00,
source_addr = b"000000",
dest_addr_ton = 0x00,
dest_addr_npi = 0x00,
destination_addr = b"000001",
esm_class = 0x00,
protocol_id = 0x00,
priority_flag = 0x00,
schedule_delivery_time = b"",
validity_period = b"",
registered_delivery = 0x01,
replace_if_present_flag = 0x00,
data_coding = 0x08,
sm_default_msg_id = 0x01,
short_message = b"MESSAGE")
assert PDU.read(BytesIO(request.serialize())) == request
c += 1
print("performance: {0:.01f} request+response packet(s)/sec.".format(c / 5.0))
def test_queue_extent_reuse():
fake_request(10.0)
def current_extents():
return sorted([
int(s.split(".")[-1])
for s in listdir(_module_state_dir(__name__))
if by_regex("^.*\\.ext_queue\\.queue\\.[0-9]+$")(s)
])
q = pmnc.state.get_queue("ext_queue",
pagesize=8192,
re_len=1024,
q_extentsize=2)
v = 0
for i in range(64): # fill up more than one extent
push(q, v)
v += 1
extents_before = current_extents()
assert len(extents_before) > 1
w = 0
t = Timeout(5.0)
while not t.expired: # keep popping and pushing to have the extents reused
push(q, v)
v += 1
assert pop(q) == w
w += 1
extents_after = current_extents()
assert extents_before[0] < extents_after[0]
def _get_modules(self):
with self._lock:
if self._modules is None: # initial state, the current directories contents is unknown
modules = self._read_modules()
self._modules = modules
elif self._settle_timeout: # a change has been detected previously and is currently being settled
if self._settle_timeout.expired:
modules = self._read_modules()
if modules != self._settle_modules: # another change occured since last time, keep settling
self._settle_timeout.reset()
self._settle_modules = modules
else: # directories contents seems to have settled
self._timeout.reset()
self._modules = modules
self._settle_modules = self._settle_timeout = None
elif self._timeout.expired: # cached contents is refreshed
self._timeout.reset()
modules = self._read_modules()
if modules != self._modules: # change detected, switch to settling
if self._settle_timeout_sec > 0.0:
self._settle_modules = modules
self._settle_timeout = Timeout(
self._settle_timeout_sec)
else:
self._modules = modules
return self._modules
def test_WorkSources_timeout():
current_thread().stopped = lambda: False
wss = WorkSources(1, 30.0)
t = Timeout(10.0)
assert wss.begin_work(3.0) == (False, None)
assert abs(t.remain - 7.0) < 1.0
def test_performance():
pmnc.log("begin performance test")
# create table
t = "table_{0:s}".format(random_string(8))
fake_request(10.0)
pmnc.transaction.postgresql_1.execute(
"CREATE TABLE {t} (id int PRIMARY KEY, key char(8) UNIQUE NOT NULL, "
"pad varchar(200) NOT NULL)".format(t = t))
# populate table
start = time()
for i in range(10):
fake_request(10.0)
sqls, params = [], {}
for j in range(100):
id = i * 100 + j
params["id{0:d}".format(id)] = id
params["key{0:d}".format(id)] = str(id)
params["pad{0:d}".format(id)] = random_string(200)
sql = "INSERT INTO {t} VALUES ({{id{id}}}, {{key{id}}}, {{pad{id}}})".format(t = t, id = id)
sqls.append(sql)
pmnc.transaction.postgresql_1.execute(*sqls, **params)
pmnc.log("{0:.01f} insert(s)/sec".format(1000 / (time() - start)))
# query table
stop = Timeout(10.0)
count = InterlockedCounter()
def th_proc():
while not stop.expired:
fake_request(10.0)
key = randint(0, 999)
rs = pmnc.transaction.postgresql_1.execute(
"SELECT id FROM {t} WHERE key = {{key}}".format(t = t), key = str(key))
assert rs[0][0]["id"] == key
count.next()
ths = [ Thread(target = th_proc) for i in range(5) ]
for th in ths: th.start()
for th in ths: th.join()
pmnc.log("{0:.01f} select(s)/sec".format(count.next() / 10.0))
# drop table
fake_request(10.0)
pmnc.transaction.postgresql_1.execute("DROP TABLE {t}".format(t = t))
pmnc.log("end performance test")
def start(self):
self._bind_timeout = Timeout(self._connect_timeout)
self._connect(self._bind_timeout.remain)
self._start_threads()
try:
self._wait_for_bind()
except:
self.stop()
raise
def disconnect(self):
try:
try:
self._sync_adapter_command("EXIT")
finally:
self._stop_adapter(Timeout(min(5.0, pmnc.request.remain)))
except:
pmnc.log.error(exc_string()) # log and ignore
finally:
TransactionalResource.disconnect(self)
def _poll_up_down_queue(self, timeout: float) -> bool: # returns "should keep running"
poll_timeout = Timeout(timeout)
while not poll_timeout.expired:
pop_timeout = Timeout(min(poll_timeout.remain, 1.0))
while not pop_timeout.expired:
event = pop_timeout.pop(self._up_down_queue)
if event is not None:
try:
node, cage, up_down, *args = event
if up_down == "up":
location, probe_result = args
# add the cage to cages known to be up and schedule
# application notification call if it was down or
# returned a different probe result
cage_info = self._up_cages.setdefault(cage, {}).setdefault(node, {})
if not cage_info or cage_info["probe_result"] != probe_result:
self._schedule_up_down_event(node, cage, "up", probe_result)
cage_info.update(location = location, probe_result = probe_result)
elif up_down == "down":
# remove the cage from cages known to be up and schedule
# application notification call it was up
if self._up_cages.setdefault(cage, {}).pop(node, None):
self._schedule_up_down_event(node, cage, "down")
except:
pmnc.log.error(exc_string()) # log and ignore
if current_thread().stopped():
return False
return True
def _poll_up_down_queue(self, timeout: float) -> bool: # returns "should keep running"
poll_timeout = Timeout(timeout)
while not poll_timeout.expired:
pop_timeout = Timeout(min(poll_timeout.remain, 1.0))
while not pop_timeout.expired:
event = pop_timeout.pop(self._up_down_queue)
if event is not None:
try:
node, cage, up_down, *args = event
if up_down == "up":
location, probe_result = args
# add the cage to cages known to be up and schedule
# application notification call if it was down or
# returned a different probe result
cage_info = self._up_cages.setdefault(cage, {}).setdefault(node, {})
if not cage_info or cage_info["probe_result"] != probe_result:
self._schedule_up_down_event(node, cage, "up", probe_result)
cage_info.update(location=location, probe_result=probe_result)
elif up_down == "down":
# remove the cage from cages known to be up and schedule
# application notification call it was up
if self._up_cages.setdefault(cage, {}).pop(node, None):
self._schedule_up_down_event(node, cage, "down")
except:
pmnc.log.error(exc_string()) # log and ignore
if current_thread().stopped():
return False
return True
def _async_request(self, rq: MongoDB_Request):
pmnc.log.info(">> {0:s}".format(rq))
try:
self._connection.async_request(rq, Timeout(pmnc.request.remain))
gle_rq = OP_QUERY("{0:s}.$cmd".format(self._database),
{"getlasterror": 1})
gle_rs = self._connection.sync_request(
gle_rq, Timeout(pmnc.request.remain))
MongoDB_Connection._command_check(gle_rs)
d = gle_rs.documents[0]
err = d.get("err")
if err: raise MongoDB_Error(err, d.get("code"))
except MongoDB_Error as e:
pmnc.log.warning("<< {0:s} !! {1:s}".format(rq, exc_string()))
ResourceError.rethrow(code=e.code,
description=str(e),
terminal=e.code is not None)
except Exception as e:
pmnc.log.warning("<< {0:s} !! {1:s}".format(rq, exc_string()))
ResourceError.rethrow(description=str(e))
else:
pmnc.log.info("<< OK")
def test_queue_many_items():
fake_request(60.0)
q = pmnc.state.get_queue("queue_many_items", re_len=64)
counter = InterlockedCounter()
def _push_n(txn, n):
for i in range(n):
c = counter.next()
_push(txn, q, "test-{0:d}".format(c), pickle)
def push_n(n):
pmnc.state.implicit_transaction(_push_n, n)
# push as much as we can per single transaction
for i in range(8):
push_n(1024)
push_n(2048)
push_n(4096)
push_n(8100)
# until we hit the limit of max_objects (8192) for one transaction
with expected(MemoryError):
push_n(8192)
Timeout(4.0).wait() # wait for checkpoint
# now pop everything off the queue
def _pop_n(txn, n):
for i in range(n):
assert _pop(txn, q, None, unpickle) is not None
def pop_n(n):
pmnc.state.implicit_transaction(_pop_n, n)
assert peek(q) == "test-0"
for i in range(8):
pop_n(8100)
pop_n(4096)
pop_n(2048)
pop_n(1024)
# the attempt to push 8192 records should have left no trace
assert peek(q) is None
class _WorkSource:
@typecheck
def __init__(self, idle_timeout: float):
self._idle_timeout = Timeout(idle_timeout)
self._has_work = False
self._working = False
def add_work(self):
self._has_work = True
def begin_work(self):
if not self._working and (self._has_work or self._idle_timeout.expired):
self._has_work = False
self._working = True
return True
else:
return False
def end_work(self):
assert self._working
self._working = False
self._idle_timeout.reset()
class _WorkSource:
@typecheck
def __init__(self, idle_timeout: float):
self._idle_timeout = Timeout(idle_timeout)
self._has_work = False
self._working = False
def add_work(self):
self._has_work = True
def begin_work(self):
if not self._working and (self._has_work
or self._idle_timeout.expired):
self._has_work = False
self._working = True
return True
else:
return False
def end_work(self):
assert self._working
self._working = False
self._idle_timeout.reset()
def test_WorkSources_stop_thread():
th_started = Event()
def th_proc():
wss = WorkSources(1, 30.0)
th_started.set()
assert wss.begin_work(10.0) == (True, None)
th = HeavyThread(target=th_proc)
th.start()
t = Timeout(30.0)
th_started.wait()
sleep(1.0)
th.stop()
assert t.remain > 25.0
def test_WorkSources_signal_kept():
current_thread().stopped = lambda: False
wss = WorkSources(4, 30.0)
wss.add_work(0)
wss.add_work(1)
wss.add_work(2)
wss.add_work(3)
t = Timeout(10.0)
assert wss.begin_work(1.0) == (False, 0)
assert wss.begin_work(1.0) == (False, 1)
assert wss.begin_work(1.0) == (False, 2)
assert wss.begin_work(1.0) == (False, 3)
assert t.remain > 9.0
assert wss.begin_work(3.0) == (False, None)
assert t.remain < 8.0
def test_InflightRequests():
ir = InflightRequests()
ir._request_ttl = timedelta(seconds = 3)
ir._cleanup_timeout = Timeout(1.0)
pdu = EnquireLinkPDU.create()
ir.add(pdu)
assert ir.remove(pdu) is pdu
assert ir.remove(pdu) is None
ir.add(pdu)
assert ir.remove(pdu.create_response()) is pdu
ir.add(pdu)
sleep(5.0)
assert ir.remove(pdu) is None
def begin_work(self,
timeout: optional(float) = None) -> (bool, optional(int)):
timeout = Timeout(timeout or self._idle_timeout + 1.0)
while not timeout.expired:
if current_thread().stopped():
return True, None
self._signal.wait(
min(timeout.remain,
3.0)) # this may spend waiting slightly less, but it's ok
with self._lock:
for i, source in enumerate(self._sources):
if source.begin_work():
return False, i
else:
self._signal.clear()
else:
return False, None
def test_WorkSources_signal_kick():
th_started = Event()
th_got_work = Event()
wss = WorkSources(1, 30.0)
def th_proc():
th_started.set()
assert wss.begin_work(10.0) == (False, 0)
th_got_work.set()
th = HeavyThread(target=th_proc)
th.start()
th_started.wait()
t = Timeout(30.0)
wss.add_work(0)
th_got_work.wait()
assert t.remain > 29.0
th.stop()
def _get_db(self, name, type, **db_opts):
with self._db_cache_lock:
db = self._db_cache.get(name)
if db is None:
db = bsddb.DB(self._env)
for n, v in db_opts.items():
getattr(db, "set_{0:s}".format(n))(v)
t = Timeout(3.0) # arbitrary timeout value
while not t.expired:
try:
db.open(os_path.join(self._dir, name), None, type,
bsddb.DB_CREATE | bsddb.DB_THREAD)
except bsddb.DBLockNotGrantedError:
_sleep_before_retry()
else:
break # success
else:
raise Exception("timed out waiting for db.open")
self._db_cache[name] = db
return db
def test_log_truncation():
def current_logs():
return sorted([
int(s[4:]) for s in listdir(_module_state_dir(__name__))
if s.startswith("log.")
])
def grow_log():
for i in range(32):
pmnc.state.set(str(randint(0, 127)), urandom(65536))
return current_logs()
fake_request(60.0)
logs_before = grow_log()
logs_after = grow_log()
while logs_before[0] == logs_after[0]:
Timeout(1.0).wait()
logs_after = grow_log()
assert logs_before[0] < logs_after[0]
def _sync_request(self,
rq: MongoDB_Request,
check: optional(callable) = None) -> MongoDB_Response:
pmnc.log.info(">> {0:s}".format(rq))
try:
rs = self._connection.sync_request(rq,
Timeout(pmnc.request.remain))
if rs.cursor_not_found:
raise MongoDB_Error("cursor not found")
if check: check(rs)
except MongoDB_Error as e:
pmnc.log.warning("<< {0:s} !! {1:s}".format(rq, exc_string()))
ResourceError.rethrow(code=e.code,
description=str(e),
terminal=e.code is not None)
except Exception as e:
pmnc.log.warning("<< {0:s} !! {1:s}".format(rq, exc_string()))
ResourceError.rethrow(description=str(e))
else:
pmnc.log.info("<< OK, {0:s}".format(rs))
return rs
def __init__(self, idle_timeout: float):
self._idle_timeout = Timeout(idle_timeout)
self._has_work = False
self._working = False
class ModuleLocator:
@typecheck
def __init__(self, cage_directory: os_path.isdir, cache_timeout: float,
settle_timeout: float):
self._cage_directory = os_path.normpath(cage_directory)
shared_directory = os_path.normpath(
os_path.join(cage_directory, "..", ".shared"))
self._shared_directory = os_path.isdir(
shared_directory) and shared_directory or None
self._timeout = Timeout(cache_timeout)
self._settle_timeout_sec = settle_timeout
self._modules = self._settle_modules = self._settle_timeout = None
self._lock = Lock()
###################################
@staticmethod
def _listdir(s):
try:
return listdir(s)
except:
return []
###################################
def _read_modules(self):
modules = {
module_name: os_path.join(self._shared_directory, module_name)
for module_name in self._listdir(self._shared_directory)
}
modules.update({
module_name: os_path.join(self._cage_directory, module_name)
for module_name in self._listdir(self._cage_directory)
})
return modules
###################################
def _get_modules(self):
with self._lock:
if self._modules is None: # initial state, the current directories contents is unknown
modules = self._read_modules()
self._modules = modules
elif self._settle_timeout: # a change has been detected previously and is currently being settled
if self._settle_timeout.expired:
modules = self._read_modules()
if modules != self._settle_modules: # another change occured since last time, keep settling
self._settle_timeout.reset()
self._settle_modules = modules
else: # directories contents seems to have settled
self._timeout.reset()
self._modules = modules
self._settle_modules = self._settle_timeout = None
elif self._timeout.expired: # cached contents is refreshed
self._timeout.reset()
modules = self._read_modules()
if modules != self._modules: # change detected, switch to settling
if self._settle_timeout_sec > 0.0:
self._settle_modules = modules
self._settle_timeout = Timeout(
self._settle_timeout_sec)
else:
self._modules = modules
return self._modules
###################################
@typecheck
def locate(self, module_name: by_regex("^[A-Za-z0-9_-]{1,128}\\.pyc?$")):
return self._get_modules().get(module_name)
def g(txn):
db2.append(pickle("item_2"), txn)
Timeout(3.0).wait()
db1.put(b"key", b"value_2", txn)
def _sleep_before_retry(): # respects wall-time timeout, see issue9892
Timeout(0.05).wait() # inherits Timeout's behaviour
def connect(self):
TransactionalResource.connect(self)
connect_timeout = Timeout(
min(self._connect_timeout, pmnc.request.remain))
self._connection.connect(connect_timeout)
self._attrs = []
