def test_row_counts():
# this is imported here rather than at module level because
# it isn't available in a plain parsl install, so this module
# would otherwise fail to import and break even a basic test
# run.
import sqlalchemy
from parsl.tests.configs.htex_local_alternate import fresh_config
if os.path.exists("monitoring.db"):
logger.info("Monitoring database already exists - deleting")
os.remove("monitoring.db")
logger.info("loading parsl")
parsl.load(fresh_config())
logger.info("invoking and waiting for result")
assert this_app().result() == 5
logger.info("cleaning up parsl")
parsl.dfk().cleanup()
parsl.clear()
# at this point, we should find one row in the monitoring database.
logger.info("checking database content")
engine = sqlalchemy.create_engine("sqlite:///monitoring.db")
with engine.begin() as connection:
result = connection.execute("SELECT COUNT(*) FROM workflow")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM task")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM try")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM status, try "
"WHERE status.task_id = try.task_id "
"AND status.task_status_name='exec_done' "
"AND task_try_time_running is NULL")
(c, ) = result.first()
assert c == 0
# Two entries: one showing manager active, one inactive
result = connection.execute("SELECT COUNT(*) FROM node")
(c, ) = result.first()
assert c == 2
# There should be one block polling status
# local provider has a status_polling_interval of 5s
result = connection.execute("SELECT COUNT(*) FROM block")
(c, ) = result.first()
assert c >= 2
logger.info("all done")
def test_row_counts():
# this is imported here rather than at module level because
# it isn't available in a plain parsl install, so this module
# would otherwise fail to import and break even a basic test
# run.
import sqlalchemy
from parsl.tests.configs.htex_local_alternate import fresh_config
if os.path.exists("monitoring.db"):
logger.info("Monitoring database already exists - deleting")
os.remove("monitoring.db")
logger.info("loading parsl")
parsl.load(fresh_config())
logger.info("invoking and waiting for result")
assert this_app().result() == 5
logger.info("cleaning up parsl")
parsl.dfk().cleanup()
parsl.clear()
# at this point, we should find one row in the monitoring database.
logger.info("checking database content")
engine = sqlalchemy.create_engine("sqlite:///monitoring.db")
with engine.begin() as connection:
result = connection.execute("SELECT COUNT(*) FROM workflow")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM task")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM try")
(c, ) = result.first()
assert c == 1
logger.info("all done")
def test_hashsum():
import sqlalchemy
from parsl.tests.configs.htex_local_alternate import fresh_config
if os.path.exists("monitoring.db"):
logger.info("Monitoring database already exists - deleting")
os.remove("monitoring.db")
logger.info("loading parsl")
parsl.load(fresh_config())
logger.info("invoking and waiting for result (1/4)")
f1 = this_app(4)
assert f1.result() == 5
logger.info("invoking and waiting for result (2/4)")
f2 = this_app(17)
assert f2.result() == 18
logger.info("invoking and waiting for result (3/4)")
f3 = this_app(4)
assert f3.result() == 5
logger.info("invoking and waiting for result (4/4)")
f4 = this_app(4)
assert f4.result() == 5
assert f1.task_def['hashsum'] == f3.task_def['hashsum']
assert f1.task_def['hashsum'] == f4.task_def['hashsum']
assert f1.task_def['hashsum'] != f2.task_def['hashsum']
logger.info("cleaning up parsl")
parsl.dfk().cleanup()
parsl.clear()
# at this point, we should find one row in the monitoring database.
logger.info("checking database content")
engine = sqlalchemy.create_engine("sqlite:///monitoring.db")
with engine.begin() as connection:
# we should have three tasks, but with only two tries, because the
# memo try should be missing
result = connection.execute("SELECT COUNT(*) FROM task")
(task_count, ) = result.first()
assert task_count == 4
# this will check that the number of task rows for each hashsum matches the above app invocations
result = connection.execute(f"SELECT COUNT(task_hashsum) FROM task WHERE task_hashsum='{f1.task_def['hashsum']}'")
(hashsum_count, ) = result.first()
assert hashsum_count == 3
result = connection.execute(f"SELECT COUNT(task_hashsum) FROM task WHERE task_hashsum='{f2.task_def['hashsum']}'")
(hashsum_count, ) = result.first()
assert hashsum_count == 1
result = connection.execute("SELECT COUNT(*) FROM status WHERE task_status_name='exec_done'")
(memo_count, ) = result.first()
assert memo_count == 2
result = connection.execute("SELECT COUNT(*) FROM status WHERE task_status_name='memo_done'")
(memo_count, ) = result.first()
assert memo_count == 2
logger.info("all done")
def test_row_counts():
from parsl.tests.configs.htex_local_alternate import fresh_config
import sqlalchemy
if os.path.exists("monitoring.db"):
logger.info("Monitoring database already exists - deleting")
os.remove("monitoring.db")
logger.info("loading parsl")
parsl.load(fresh_config())
logger.info("invoking apps and waiting for result")
assert this_app().result() == 5
assert this_app().result() == 5
# now we've run some apps, send fuzz into the monitoring ZMQ
# socket, before trying to run some more tests.
# there are different kinds of fuzz:
# could send ZMQ messages that are weird
# could send random bytes to the TCP socket
# the latter is what i'm most suspicious of in my present investigation
# dig out the interchange port...
hub_address = parsl.dfk().hub_address
hub_interchange_port = parsl.dfk().hub_interchange_port
# this will send a string to a new socket connection
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((hub_address, hub_interchange_port))
s.sendall(b'fuzzing\r')
# this will send a non-object down the DFK's existing ZMQ connection
parsl.dfk().monitoring._dfk_channel.send(b'FuzzyByte\rSTREAM')
# This following attack is commented out, because monitoring is not resilient
# to this.
# In practice, it works some of the time but in some circumstances,
# it would still abandon writing multiple unrelated records to the database,
# causing ongoing monitoring data loss.
# This will send an unusual python object down the
# DFK's existing ZMQ connection. this doesn't break the router,
# but breaks the db_manager in a way that isn't reported until
# the very end of the run, and database writing is abandoned
# rather than completing, in this case.
# I'm unclear if this is a case we should be trying to handle.
# parsl.dfk().monitoring._dfk_channel.send_pyobj("FUZZ3")
# hopefully long enough for any breakage to happen
# before attempting to run more tasks
time.sleep(5)
assert this_app().result() == 5
assert this_app().result() == 5
logger.info("cleaning up parsl")
parsl.dfk().cleanup()
parsl.clear()
# at this point, we should find one row in the monitoring database.
logger.info("checking database content")
engine = sqlalchemy.create_engine("sqlite:///monitoring.db")
with engine.begin() as connection:
result = connection.execute("SELECT COUNT(*) FROM workflow")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM task")
(c, ) = result.first()
assert c == 4
result = connection.execute("SELECT COUNT(*) FROM try")
(c, ) = result.first()
assert c == 4
logger.info("all done")
def test_row_counts():
from parsl.tests.configs.htex_local_alternate import fresh_config
import sqlalchemy
if os.path.exists("monitoring.db"):
logger.info("Monitoring database already exists - deleting")
os.remove("monitoring.db")
engine = sqlalchemy.create_engine("sqlite:///monitoring.db")
logger.info("loading parsl")
parsl.load(fresh_config())
# parsl.load() returns before all initialisation of monitoring
# is complete, which means it isn't safe to take a read lock on
# the database yet. This delay tries to work around that - some
# better async behaviour might be nice, but what?
#
# Taking a read lock before monitoring is initialized will cause
# a failure in the part of monitoring which creates tables, and
# which is not protected against read locks at the time this test
# was written.
time.sleep(10)
# to get an sqlite3 read lock that is held over a controllable
# long time, create a transaction and perform a SELECT in it.
# The lock will be held until the end of the transaction.
# (see bottom of https://sqlite.org/lockingv3.html)
logger.info("Getting a read lock on the monitoring database")
with engine.begin() as readlock_connection:
readlock_connection.execute("BEGIN TRANSACTION")
result = readlock_connection.execute("SELECT COUNT(*) FROM workflow")
(c, ) = result.first()
assert c == 1
# now readlock_connection should have a read lock that will
# stay locked until the transaction is ended, or the with
# block ends.
logger.info("invoking and waiting for result")
assert this_app().result() == 5
# there is going to be some raciness here making sure that
# the database manager actually tries to write while the
# read lock is held. I'm not sure if there is a better way
# to detect this other than a hopefully long-enough sleep.
time.sleep(10)
logger.info("cleaning up parsl")
parsl.dfk().cleanup()
parsl.clear()
# at this point, we should find data consistent with executing one
# task in the database.
logger.info("checking database content")
with engine.begin() as connection:
result = connection.execute("SELECT COUNT(*) FROM workflow")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM task")
(c, ) = result.first()
assert c == 1
result = connection.execute("SELECT COUNT(*) FROM try")
(c, ) = result.first()
assert c == 1
logger.info("all done")