def test_cross_db_deadlock():
fake_request(30.0)
db1 = pmnc.state.get_database("db1")
db2 = pmnc.state.get_queue("db2", re_len=6144)
def f(txn):
db1.put(b"key", b"value_1", txn)
Timeout(3.0).wait()
db2.append(pickle("item_1"), txn)
def g(txn):
db2.append(pickle("item_2"), txn)
Timeout(3.0).wait()
db1.put(b"key", b"value_2", txn)
th_f = HeavyThread(target=lambda: pmnc.state.implicit_transaction(f))
th_g = HeavyThread(target=lambda: pmnc.state.implicit_transaction(g))
th_f.start()
th_g.start()
th_f.join()
th_g.join()
# now see what went through
def fetch_result(txn):
value = db1.get(b"key", None, txn)
item1 = _pop(txn, db2, None, unpickle)
item2 = _pop(txn, db2, None, unpickle)
return value, item1, item2
value, item1, item2 = pmnc.state.implicit_transaction(fetch_result)
assert value in (b"value_1", b"value_2")
assert (item1, item2) in (("item_1", "item_2"), ("item_2", "item_1"))
def test_cross_db_deadlock():
fake_request(30.0)
db1 = pmnc.state.get_database("db1")
db2 = pmnc.state.get_queue("db2", re_len=6144)
def f(txn):
db1.put(b"key", b"value_1", txn)
Timeout(3.0).wait()
db2.append(pickle("item_1"), txn)
def g(txn):
db2.append(pickle("item_2"), txn)
Timeout(3.0).wait()
db1.put(b"key", b"value_2", txn)
th_f = HeavyThread(target=lambda: pmnc.state.implicit_transaction(f))
th_g = HeavyThread(target=lambda: pmnc.state.implicit_transaction(g))
th_f.start()
th_g.start()
th_f.join()
th_g.join()
# now see what went through
def fetch_result(txn):
value = db1.get(b"key", None, txn)
item1 = _pop(txn, db2, None, unpickle)
item2 = _pop(txn, db2, None, unpickle)
return value, item1, item2
value, item1, item2 = pmnc.state.implicit_transaction(fetch_result)
assert value in (b"value_1", b"value_2")
assert (item1, item2) in (("item_1", "item_2"), ("item_2", "item_1"))
def secondary_startup(node, cage, mode):
cage_dir = os_path.join(cages_dir, cage)
logs_dir = os_path.join(cage_dir, "logs")
lib_dir = os_path.join(cage_dir, "lib")
sys_path.insert(0, lib_dir)
log_abbrevs = {
1: "ERR",
2: "MSG",
3: "WRN",
4: "LOG",
5: "INF",
6: "DBG",
7: "NSE"
}
log_encoding = "windows-1251"
log_translate = b" \t " + bytes(range(
32, 256))
log_lock = Lock()
log_yyyymmdd = None
log_file = None
# the following function will serve for all cage's logging
def log(message, *, msg_level):
nonlocal log_yyyymmdd, log_file
line_time = time()
line_yyyymmdd, line_hhmmss = strftime("%Y%m%d %H:%M:%S",
localtime(line_time)).split(" ")
log_line = "{0:s}.{1:02d} {2:s} [{3:s}] {4:s}".format(
line_hhmmss,
int(line_time * 100) % 100, log_abbrevs.get(msg_level, "???"),
current_thread().name, message)
with log_lock:
if line_yyyymmdd != log_yyyymmdd: # rotate log file
try:
new_log_file_name = os_path.join(
logs_dir,
"{0:s}-{1:s}.log".format(cage, line_yyyymmdd))
new_log_file = fopen(
new_log_file_name,
os.O_WRONLY | os.O_CREAT | os.O_APPEND)
except:
pass # if rotation fails, previous log file will still be used
else:
try:
close(log_file)
except:
pass # this also catches the attempt to close None
log_file = new_log_file
log_yyyymmdd = line_yyyymmdd
if log_file is not None:
if message:
write(
log_file,
log_line.encode(log_encoding,
"replace").translate(log_translate) +
b"\n")
if msg_level == 1:
fsync(log_file)
###################################
# create loader instance using initial default logging level
pmnc = ModuleLoader(node, cage, cage_dir, log, "LOG", 2.0, 1.0)
###################################
current_thread().name = "startup"
if mode == "NORMAL":
log("the cage is starting up", msg_level=2)
elif mode == "FAILURE":
log("the cage is restarting after a failure", msg_level=2)
###################################
if win32_com:
_main_thread_id = GetCurrentThreadId()
def cage_thread_proc():
try:
pmnc.startup.start()
try:
while not pmnc.startup.wait(3.0):
pmnc.startup.maintenance()
except:
pmnc.log.error(exc_string()) # log and ignore
finally:
pmnc.startup.stop()
finally:
if win32_com: # release the main thread blocked in PumpMessages
PostThreadMessage(_main_thread_id, WM_QUIT)
cage_thread = HeavyThread(target=cage_thread_proc, name="cage")
cage_thread.start()
###################################
def termination_watchdog_proc():
try:
while stdout.write("\n") > 0:
stdout.flush()
sleep(3.0)
except:
pass
finally:
pmnc.startup.exit()
termination_watchdog = HeavyThread(target=termination_watchdog_proc,
name="stdout")
termination_watchdog.start()
###################################
# wait for the cage thread to detect shutdown and terminate
if win32_com:
PumpMessages() # in the meanwhile become a message pump
cage_thread.join()
###################################
log("the cage has been properly shut down", msg_level=2)
log("", msg_level=1) # force flush of a log file
