def register_parallel_pytest_cov():
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
def manager_proc(ipc_dir):
try:
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
except ImportError:
pass
addrs = {
'results': 'ipc://%s/manager_results' % ipc_dir,
'comm': 'ipc://%s/manager_comm' % ipc_dir,
'graph': 'ipc://%s/manager_graph' % ipc_dir,
'msg': 'ipc://%s/manager_msg' % ipc_dir,
'info': 'ipc://%s/manager_info' % ipc_dir,
'export': 'ipc://%s/manager_export' % ipc_dir,
'view': 'ipc://%s/manager_view' % ipc_dir,
}
# start the manager process
proc = mp.Process(
name='manager',
target=run_manager,
args=(1, 1, addrs['results'], addrs['graph'], addrs['comm'],
addrs['msg'], addrs['info'], addrs['export'], addrs['view'],
None, None, None)
)
proc.daemon = False
proc.start()
yield addrs
# cleanup the manager process
proc.terminate()
proc.join(1)
return proc.exitcode
def run(self, q):
with ExitStack() as stack:
try:
# Handle SIGTERM by calling sys.exit(0), which will raise a SystemExit exception,
# properly executing cleanup code in `finally` clauses and context managers.
# This is required to terminate buildbox-casd on SIGTERM.
signal.signal(signal.SIGTERM,
lambda signalnum, frame: sys.exit(0))
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
server = stack.enter_context(self._create_server())
port = server.add_insecure_port("localhost:0")
server.start()
except Exception:
q.put(None)
raise
# Send port to parent
q.put(port)
# Sleep until termination by signal
signal.pause()
def broker(ipc_dir, graphmgr_addr):
try:
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
except ImportError:
pass
parent_comm, child_comm = mp.Pipe()
# start the manager process
proc = mp.Process(name='broker',
target=BrokerHelper.execute,
args=(graphmgr_addr, ipc_dir, child_comm),
daemon=False)
proc.start()
broker = BrokerProxy(parent_comm)
yield broker
# cleanup the manager process
broker.exit()
proc.join(2)
# if ami still hasn't exitted then kill it
if proc.is_alive():
proc.terminate()
proc.join()
return proc.exitcode
def _start_sub_pool(cls, actor_config: ActorPoolConfig, process_index: int,
started: multiprocessing.Event):
if not _is_windows:
try:
# register coverage hooks on SIGTERM
from pytest_cov.embed import cleanup_on_sigterm
if 'COV_CORE_SOURCE' in os.environ: # pragma: no branch
cleanup_on_sigterm()
except ImportError: # pragma: no cover
pass
conf = actor_config.get_pool_config(process_index)
suspend_sigint = conf['suspend_sigint']
if suspend_sigint:
signal.signal(signal.SIGINT, lambda *_: None)
logging_conf = conf['logging_conf'] or {}
if logging_conf.get('file'):
logging.config.fileConfig(logging_conf['file'])
elif logging_conf.get('level'):
logging.basicConfig(level=logging_conf['level'],
format=logging_conf.get('format'))
use_uvloop = conf['use_uvloop']
if use_uvloop:
import uvloop
asyncio.set_event_loop(uvloop.new_event_loop())
else:
asyncio.set_event_loop(asyncio.new_event_loop())
coro = cls._create_sub_pool(actor_config, process_index, started)
asyncio.run(coro)
def setUp(self):
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
self.barrier = Barrier(2)
MockNodeHandler.BARRIER = self.barrier
self.node_handler = node_handler_process.NodeHandlerProcess
node_handler_process.NodeHandlerProcess = MockNodeHandler
from src.backup_scheduler.backup_scheduler import BackupScheduler
shutil.rmtree('/tmp/disk_db_concus', ignore_errors=True)
os.mkdir('/tmp/disk_db_concus')
shutil.rmtree('/tmp/data_for_backup', ignore_errors=True)
os.mkdir('/tmp/data_for_backup')
with open('/tmp/data_for_backup/data', 'w') as data_file:
data_file.write("adasdsa")
database = DiskDatabase('/tmp/disk_db_concus')
shutil.rmtree('/tmp/backup_scheduler_path', ignore_errors=True)
os.mkdir('/tmp/backup_scheduler_path')
with open('/tmp/backup_scheduler_path/trash', 'w') as trash_file:
trash_file.write("trash")
backup_scheduler_recv, self.client_listener_send = Pipe(False)
self.client_listener_recv, backup_scheduler_send = Pipe(False)
backup_scheduler = BackupScheduler('/tmp/backup_scheduler_path', database,
backup_scheduler_recv, backup_scheduler_send, 10)
self.p = Process(target=backup_scheduler)
self.p.start()
def __new__(cls, *args, **kwargs):
try:
if 'COV_CORE_SOURCE' in os.environ: # pragma: no branch
# register coverage hooks on SIGTERM
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
except ImportError: # pragma: no cover
pass
return super().__new__(cls, *args, **kwargs)
def enable_code_coverage():
"""Ensure code coverage works as expected for `multiprocesses` tests.
As its harmless for other types of test, we always run this fixture."""
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
def setup_module(module):
# Here, we add compatibility for multiprocessing coverage reports.
# via: https://pytest-cov.readthedocs.io/en/latest/subprocess-support.html#if-you-use-multiprocessing-pool
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
def setUp(self) -> None:
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
self.barrier = Barrier(2)
self.p = None
def _inner():
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
app.run(
host='localhost',
port=LIVE_SERVER_PORT,
debug=False,
use_reloader=False,
threaded=False
)
def sync_proc(request, ipc_dir):
try:
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
except ImportError:
pass
interval = 10
addr = "ipc://%s/sync" % ipc_dir
comm_addr = "ipc://%s/sync-comm" % ipc_dir
if isinstance(request.param, tuple):
nclients, start = request.param
else:
nclients = request.param
start = 0
sync_proc = mp.Process(target=run_syncer,
args=(addr, comm_addr, start, interval),
name='syncer',
daemon=True)
sync_proc.start()
ctx = zmq.Context()
# socket for communicating with syncer
comm = ctx.socket(zmq.REQ)
comm.connect(comm_addr)
# sync socket to return from the fixture
syncs = []
for _ in range(nclients):
sync = ctx.socket(zmq.REQ)
sync.connect(addr)
syncs.append(sync)
yield syncs, start
# cleanup the syncer
comm.send_string('exit')
ret = comm.recv_pyobj()
# join syncer thread
sync_proc.join(2)
# if ami still hasn't exitted then kill it
if sync_proc.is_alive():
print("try killing sync_proc")
sync_proc.terminate()
sync_proc.join(1)
# cleanup zmq
for sync in syncs:
sync.close()
comm.close()
ctx.term()
return ret
def dbus_service(request):
cleanup_on_sigterm()
service = DBusObject()
try:
service.request_name(request.param)
except RuntimeError:
pytest.skip("Can't get the requested name")
try:
yield service
finally:
service.stop()
def _start_sub_pool(cls, actor_config: ActorPoolConfig, process_index: int,
started: multiprocessing.Event):
try:
# register coverage hooks on SIGTERM
from pytest_cov.embed import cleanup_on_sigterm
if 'COV_CORE_SOURCE' in os.environ: # pragma: no branch
cleanup_on_sigterm()
except ImportError: # pragma: no cover
pass
asyncio.set_event_loop(asyncio.new_event_loop())
coro = cls._create_sub_pool(actor_config, process_index, started)
asyncio.run(coro)
def dbus_service():
cleanup_on_sigterm()
name = 'com.example.object'
service = DBusObject()
try:
service.request_name(name)
except RuntimeError:
pytest.skip("Can't get the requested name")
try:
yield service
finally:
service.stop()
def test_concurrency():
ret = []
with concurrent_pool("s", ret=ret) as c:
for i in range(4):
c.submit(fn, i, bias=1)
assert list(sorted(ret)) == [1, 2, 3, 4]
ret = []
with concurrent_pool("c", ret=ret) as c:
for i in range(4):
c.submit(async_fn, i, bias=1)
assert list(sorted(ret)) == [1, 2, 3, 4]
ret = []
with concurrent_pool("t", ret=ret) as c:
for i in range(4):
c.submit(fn, i, bias=1)
assert list(sorted(ret)) == [1, 2, 3, 4]
if "Windows" in platform.architecture()[1]:
def shutdown(frame, signum):
# your app's shutdown or whatever
pass
signal.signal(signal.SIGBREAK, shutdown)
try:
from pytest_cov.embed import cleanup_on_signal
except ImportError:
pass
else:
cleanup_on_signal(signal.SIGBREAK)
return
else:
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
ret = []
with concurrent_pool("p", ret=ret) as c:
for i in range(4):
c.submit(fn, i, bias=1)
assert list(sorted(ret)) == [1, 2, 3, 4]
def flowchart(request, workerjson, broker, ipc_dir, graphmgr_addr,
qevent_loop):
try:
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
except ImportError:
pass
parser = build_parser()
args = parser.parse_args([
"-n", "1", '-i',
str(ipc_dir), '--headless',
'%s://%s' % (request.param, workerjson)
])
queue = mp.Queue()
ami = mp.Process(name='ami', target=run_ami, args=(args, queue))
ami.start()
try:
# wait for ami to be fully up before updating the sources
with GraphCommHandler(graphmgr_addr.name, graphmgr_addr.comm) as comm:
while not comm.sources:
time.sleep(0.1)
with Flowchart(broker_addr=broker.broker_sub_addr,
graphmgr_addr=graphmgr_addr,
checkpoint_addr=broker.checkpoint_pub_addr) as fc:
qevent_loop.run_until_complete(fc.updateSources(init=True))
yield (fc, broker)
except Exception as e:
# let the fixture exit 'gracefully' if it fails
print("error setting up flowchart fixture:", e)
yield None
finally:
queue.put(None)
ami.join(2)
# if ami still hasn't exitted then kill it
if ami.is_alive():
ami.terminate()
ami.join()
if ami.exitcode == 0 or ami.exitcode == -signal.SIGTERM:
return 0
else:
print('AMI exited with non-zero status code: %d' % ami.exitcode)
return 1
def setUp(self):
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
self.barrier = Barrier(2)
self.client_listener = None
self.backup_scheduler_recv, client_listener_send = Pipe(False)
client_listener_recv, self.backup_scheduler_send = Pipe(False)
self.p = Process(target=self._launch_process,
args=(client_listener_send, client_listener_recv))
self.p.start()
def deploy_server(tmpdir):
db_path = str(tmpdir.join("test.worky"))
server.app.config['STORAGE'] = Storage(db_path)
server_process = Process(target=server.app.run, args=(host_ip, ))
cleanup_on_sigterm()
server_process.start()
yield
server_process.terminate()
server_process.join()
def graphmgr_addr(ipc_dir):
try:
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
except ImportError:
pass
comm_addr = "ipc://%s/comm" % ipc_dir
view_addr = "ipc://%s/view" % ipc_dir
graphinfo_addr = "ipc://%s/info" % ipc_dir
graphmgr = GraphMgrAddress("graph", comm_addr, view_addr, graphinfo_addr)
yield graphmgr
def setUp(self) -> None:
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
self.processes_to_join = []
self.queues_to_purge = []
self.dirs_to_delete = []
self.recv_pipe, self.write_pipe = Pipe(False)
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host="localhost"))
self.channel = self.connection.channel()
self._setup_queue('pipeline_start')
def run_until_timeout(target, args, timeout=1):
# https://pytest-cov.readthedocs.io/en/v2.10.1_a/subprocess-support.html
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
p = Process(target=target, args=args)
try:
p.start()
p.join(timeout=timeout)
finally:
p.terminate()
def setUp(self) -> None:
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
shutil.rmtree('/tmp/backup_output', ignore_errors=True)
os.mkdir('/tmp/backup_output')
with open('/tmp/example', 'w') as example_file:
example_file.write("asd")
BackupFile.create_from_path('/tmp/example', "/tmp/backup_output/out2")
SidecarProcess.logger = logging.getLogger("dummy")
self.sidecar_process = SidecarProcess(TestSidecar.PORT, 3)
self.p = Process(target=self.sidecar_process)
self.p.start()
def setUp(self) -> None:
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
self.recv_pipe, self.write_pipe = Pipe(False)
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host="localhost"))
self.channel = self.connection.channel()
self.channel.queue_declare(queue=CONSUME_QUEUE)
self.channel.queue_purge(CONSUME_QUEUE)
self.test_process = None
self.consume_process = Process(target=self._read_process,
args=(self.write_pipe, ))
self.consume_process.start()
def _serve(host="", port=5000, quiet=False, use_coverage=False):
"""
Wrapper function for :func:`jsonrpcserver.serve` and :func:`.quiet_serve`. Can be forked into background.
Sets up SIGTERM hook using :py:func:`pytest_cov.embed.cleanup_on_sigterm` so coverage data is correctly
saved when the subprocess is terminated.
"""
# If this is being called from a unit test, then attempt to setup the pytest-cov SIGTERM hook to ensure
# coverage data is generated correctly for this subprocess.
if use_coverage:
try:
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
except ImportError:
warnings.warn(
"Could not import coverage module in child process...")
pass
srv = quiet_serve if quiet else serve
srv(host, port)
def run_service_datajoint_server():
# The following cleanup is needed because we terminate this compute resource process
# See: https://pytest-cov.readthedocs.io/en/latest/subprocess-support.html
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
os.environ['RUNNING_PYTEST'] = 'TRUE'
with hi.ConsoleCapture(label='[datajoint-server]'):
ss = hi.ShellScript(f"""
#!/bin/bash
set -ex
docker kill datajoint-server-fixture > /dev/null 2>&1 || true
docker rm datajoint-server-fixture > /dev/null 2>&1 || true
exec docker run --name datajoint-server-fixture -e MYSQL_ROOT_PASSWORD=tutorial -p {DATAJOINT_SERVER_PORT}:3306 datajoint/mysql
""", redirect_output_to_stdout=True)
ss.start()
ss.wait()
def setUp(self) -> None:
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
shutil.rmtree('/tmp/message_set', ignore_errors=True)
os.mkdir('/tmp/message_set')
self.message_set = DiskMessageSet('/tmp/message_set', recover_state_on_init=True)
self.recv_pipe, self.write_pipe = Pipe(False)
self.connection = pika.BlockingConnection(pika.ConnectionParameters(host="localhost"))
self.channel = self.connection.channel()
self.channel.queue_declare(queue=CONSUME_QUEUE)
self.channel.queue_declare(queue=RESPONSE_QUEUE)
self.channel.queue_purge(CONSUME_QUEUE)
self.channel.queue_purge(RESPONSE_QUEUE)
self.test_process = None
self.consume_process = Process(target=self._read_process, args=(self.write_pipe,))
self.consume_process.start()
def setUp(self) -> None:
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
self.processes_to_join = []
self.queues_to_purge = []
self.dirs_to_delete = []
self.recv_pipe, self.write_pipe = Pipe(False)
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host="localhost"))
self.channel = self.connection.channel()
self._setup_queue('yelp_businesses_news')
self._setup_queue('pipeline_result')
self._setup_queue('notify_business_load_end')
self._setup_queue('queue_to_join')
self._self_register_dir('/tmp/downloader_data')
self._self_register_dir('/tmp/joiner_data')
def start_ami(request, workerjson):
try:
from pytest_cov.embed import cleanup_on_sigterm
cleanup_on_sigterm()
except ImportError:
pass
parser = build_parser()
args = parser.parse_args([
"-n", "1", '--headless', '--tcp',
'%s://%s' % (request.param, workerjson)
])
queue = mp.Queue()
ami = mp.Process(name='ami', target=run_ami, args=(args, queue))
ami.start()
try:
host = "127.0.0.1"
comm_addr = "tcp://%s:%d" % (host, BasePort + Ports.Comm)
with GraphCommHandler(args.graph_name, comm_addr) as comm_handler:
yield comm_handler
except Exception as e:
# let the fixture exit 'gracefully' if it fails
print(e)
yield None
finally:
queue.put(None)
ami.join(1)
# if ami still hasn't exitted then kill it
if ami.is_alive():
ami.terminate()
ami.join(1)
if ami.exitcode == 0 or ami.exitcode == -signal.SIGTERM:
return 0
else:
print('AMI exited with non-zero status code: %d' % ami.exitcode)
return 1
def test_2d_bc(tmp_path):
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
sample_n = 10
worker = 4
path = tmp_path / "test"
sys.argv = [
"layout_generator",
"generate",
"--data_dir",
str(path),
"--bcs",
"[[0.01, 0], [0.02, 0]]",
"--bcs",
"[[0.08, 0], [0.09, 0]]",
"--sample_n",
str(sample_n),
"--worker",
str(worker),
]
options = main(options_flag=True)
data_dir = path
# assert data_dir == 's'
# data_path_list = os.listdir(options.data_dir)
data_path_list = list(data_dir.glob(f"*.{options.file_format}"))
assert data_dir.exists()
assert len(data_path_list) == options.sample_n
datum_path = data_path_list[0]
r = io.load_mat(datum_path)
assert set(["u", "F", "list", "xs", "ys"]).issubset(set(r.keys()))
u = r["u"]
assert u.shape == (options.nx,) * options.ndim
assert u.min() >= options.u_D
import os
import platform
import sys
from pathlib import Path
from subprocess import PIPE, run
import pytest
pytest_plugins = ['helpers_namespace']
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
# pylint: disable=E1101
# pytest.helpers is dynamic so it confuses pylint
if sys.version_info.major < 3:
print("Requires Python 3.4+")
sys.exit(1)
@pytest.helpers.register
def is_linux():
return platform.system() == 'Linux'
@pytest.helpers.register
