class TestGRPCServer:
def __init__(self, manage_py, params=None):
if params is None:
params = {}
params.setdefault('--port', 50000 + randint(0, 10000))
self.port = params.get('--port')
self.manage_py = manage_py
self.process = TCPExecutor(['python', self.manage_py, 'grpcserver'] +
list(self.flat_params(params)),
host='localhost',
port=self.port)
@classmethod
def flat_params(cls, dict_):
for k, v in dict_.items():
yield str(k)
if str(v) != "":
yield str(v)
def addr(self):
return "localhost:%s" % self.port
def start(self):
self.process.start()
def stop(self):
self.process.stop()
def test_it_raises_error_on_timeout():
"""Check if TimeoutExpired gets rised correctly."""
command = 'bash -c "sleep 10 && nc -l 3000"'
executor = TCPExecutor(command, host='localhost', port=3000, timeout=2)
with pytest.raises(TimeoutExpired):
executor.start()
assert executor.running() is False
def test_it_raises_error_on_timeout():
"""Check if TimeoutExpired gets raised correctly."""
command = 'bash -c "sleep 10 && nc -l 3000"'
executor = TCPExecutor(command, host='localhost', port=3000, timeout=2)
with pytest.raises(TimeoutExpired):
executor.start()
assert executor.running() is False
def mongo_proc_fixture(request):
"""
Mongodb process fixture.
:param FixtureRequest request: fixture request object
:rtype: mirakuru.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
tmpdir = gettempdir()
mongo_exec = executable or config['exec']
mongo_params = params or config['params']
mongo_host = host or config['host']
mongo_port = get_port(port) or get_port(config['port'])
mongo_logsdir = logsdir or config['logsdir']
mongo_logpath = os.path.join(
mongo_logsdir,
'mongo.{port}.log'.format(
port=mongo_port
)
)
mongo_db_path = os.path.join(
tmpdir,
'mongo.{port}'.format(
port=mongo_port
)
)
os.mkdir(mongo_db_path)
request.addfinalizer(
lambda: os.path.exists(mongo_db_path) and rmtree(mongo_db_path)
)
mongo_executor = TCPExecutor(
(
'{mongo_exec} --bind_ip {host} --port {port}'
' --dbpath {dbpath} --logpath {logpath} {params}'
).format(
mongo_exec=mongo_exec,
params=mongo_params,
host=mongo_host,
port=mongo_port,
dbpath=mongo_db_path,
logpath=mongo_logpath,
),
host=mongo_host,
port=mongo_port,
timeout=60
)
mongo_executor.start()
request.addfinalizer(mongo_executor.stop)
return mongo_executor
def __init__(self, manage_py, params=None):
if params is None:
params = {}
params.setdefault('--port', 50000 + randint(0, 10000))
self.port = params.get('--port')
self.manage_py = manage_py
self.process = TCPExecutor(['python', self.manage_py, 'grpcserver'] +
list(self.flat_params(params)),
host='localhost',
port=self.port)
def test_fail_if_other_executor_running():
"""Test raising AlreadyRunning exception."""
executor = TCPExecutor(http_server_cmd, host='localhost', port=PORT)
executor2 = TCPExecutor(http_server_cmd, host='localhost', port=PORT)
with executor:
assert executor.running() is True
with pytest.raises(AlreadyRunning):
executor2.start()
with pytest.raises(AlreadyRunning):
with executor2:
pass
def test_default_port():
"""
Test default port for the base TCP check.
Check if HTTP executor fills in the default port for the TCP check
from the base class if no port is provided in the URL.
"""
executor = HTTPExecutor(http_server_cmd, 'http://{0}/'.format(HOST))
assert executor.url.port is None
assert executor.port == PORT
assert TCPExecutor.pre_start_check(executor) is False
executor.start()
assert TCPExecutor.pre_start_check(executor) is True
executor.stop()
def test_default_port():
"""
Test default port for the base TCP check.
Check if HTTP executor fills in the default port for the TCP check
from the base class if no port is provided in the URL.
"""
executor = HTTPExecutor(http_server_cmd, 'http://{}/'.format(HOST))
assert executor.url.port is None
assert executor.port == PORT
assert TCPExecutor.pre_start_check(executor) is False
executor.start()
assert TCPExecutor.pre_start_check(executor) is True
executor.stop()
def test_default_port():
"""
Test default port for the base TCP check.
Check if HTTP executor fills in the default port for the TCP check
from the base class if no port is provided in the URL.
"""
executor = HTTPExecutor(HTTP_NORMAL_CMD, f'http://{HOST}/')
assert executor.url.port is None
assert executor.port == PORT
assert TCPExecutor.pre_start_check(executor) is False
executor.start()
assert TCPExecutor.pre_start_check(executor) is True
executor.stop()
def s3_client(monkeypatch, mocker, config):
import botocore.session
from mirakuru import TCPExecutor
host = '127.0.0.1'
socket, port = tornado.testing.bind_unused_port()
socket.close()
monkeypatch.setenv('TEST_SERVER_MODE', 'true')
monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', '')
monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'test-key')
monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'test-secret-key')
monkeypatch.setenv('AWS_SESSION_TOKEN', 'test-session-token')
endpoint_url = "http://%s:%s" % (host, port)
config.TC_AWS_ENDPOINT = endpoint_url
mocker.patch('subprocess.Popen', side_effect=SubprocessStderrPipe)
executor = TCPExecutor((sys.executable, '-m', 'moto.server', '-H', host,
'-p', str(port), "s3"),
host=host,
port=port,
timeout=10)
try:
with executor:
yield botocore.session.get_session().create_client(
's3', endpoint_url=endpoint_url)
finally:
if asyncio is not None:
event_loop = asyncio.get_event_loop()
instances = getattr(Bucket, "_instances", None) or {}
for bucket in instances.values():
event_loop.run_until_complete(bucket._client.close())
Bucket._instances = {}
def test_start_and_wait(caplog: LogCaptureFixture):
"""Test if executor await for process to accept connections."""
caplog.set_level(logging.DEBUG, logger="mirakuru")
executor = TCPExecutor(NC_COMMAND, "localhost", port=3000, timeout=5)
executor.start()
assert executor.running() is True
executor.stop()
def dynamodb_proc_fixture(request):
"""
Process fixture for DynamoDB.
It starts DynamoDB when first used and stops it at the end
of the tests. Works on ``DynamoDBLocal.jar``.
:param FixtureRequest request: fixture request object
:rtype: pytest_dbfixtures.executors.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
path_dynamodb_jar = os.path.join((dynamodb_dir or config['dir']),
'DynamoDBLocal.jar')
if not os.path.isfile(path_dynamodb_jar):
raise JarPathException(
'You have to provide a path to the dir with dynamodb jar file.'
)
dynamodb_port = get_port(port or config['port'])
dynamodb_delay = delay or config['delay']
dynamodb_host = host or config['host']
dynamodb_executor = TCPExecutor(
'''java
-Djava.library.path=./DynamoDBLocal_lib
-jar {path_dynamodb_jar}
-inMemory
{delay}
-port {port}'''.format(
path_dynamodb_jar=path_dynamodb_jar,
port=dynamodb_port,
delay='-delayTransientStatuses' if dynamodb_delay else '',
),
host=dynamodb_host,
port=dynamodb_port,
timeout=60,
)
dynamodb_executor.start()
request.addfinalizer(dynamodb_executor.stop)
return dynamodb_executor
def oauth_redirect_server(temporary_path):
config = temporary_path / 'config.yml'
database = temporary_path / 'registered_clients'
auth = uuid4().hex
with config.open('w') as f:
f.write('\n'.join(
('database: {}'.format(database), 'auth: {}'.format(auth))))
command = 'oauth-redirect -h 127.0.0.1 -p 8000 -a {} -d {}'.format(
auth, database)
server = TCPExecutor(command, host='127.0.0.1', port=8000, shell=True)
server.start()
server.config = config
server.database = database
server.auth = auth
server.url = 'http://127.0.0.1:8000'
yield server
server.stop()
def moto_service():
executor = TCPExecutor(
('moto_server', '-H', host, '-p', str(port), service),
host=host,
port=port,
timeout=timeout)
connection_params.setdefault('proxy', host)
connection_params.setdefault('proxy_port', port)
connection_params.setdefault('aws_access_key_id', '')
connection_params.setdefault('aws_secret_access_key', '')
connection_params.setdefault('is_secure', False)
with executor:
yield get_connection(service, connection_params)
def __init__(self, path='/tmp/dynamodb'):
self._path = path
self._path_dynamodb_jar = os.path.join(path, 'DynamoDBLocal.jar')
self._port = self._get_open_port()
self.executor = TCPExecutor(
f'java -Djava.library.path=./DynamoDBLocal_lib -jar {self._path_dynamodb_jar} -inMemory -port {self._port}',
host='localhost',
port=self._port,
timeout=60,
)
# Write random credentials into env
self.aws_access_key = str(uuid4())
self.aws_secret_access_key = str(uuid4())
self.region = str(uuid4())
os.environ['AWS_ACCESS_KEY_ID'] = self.aws_access_key
os.environ['AWS_SECRET_ACCESS_KEY'] = self.aws_secret_access_key
os.environ['AWS_DEFAULT_REGION'] = self.region
self.__resources = set()
def test_start_and_wait(timeout):
"""Test if executor await for process to accept connections."""
command = 'bash -c "sleep 2 && nc -l 3000"'
executor = TCPExecutor(command, 'localhost', port=3000, timeout=timeout)
executor.start()
assert executor.running() is True
executor.stop()
def test_start_and_wait():
"""Test if executor await for process to accept connections."""
command = 'bash -c "sleep 2 && nc -l 3000"'
executor = TCPExecutor(command, 'localhost', port=3000, timeout=5)
executor.start()
assert executor.running() is True
executor.stop()
# check proper __str__ and __repr__ rendering:
assert 'TCPExecutor' in repr(executor)
assert command in str(executor)
def memcached_server():
path = shutil.which('memcached')
if not path:
raise RuntimeError("Could not find memcached executable")
host = '127.0.0.1'
port = port_for.select_random()
executor = TCPExecutor(f'memcached -l {host} -p {port}', host, port)
executor.start()
yield executor
executor.stop()
executor.kill()
def mongo_proc_fixture(request):
"""
Mongodb process fixture.
:param FixtureRequest request: fixture request object
:rtype: mirakuru.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
tmpdir = gettempdir()
mongo_exec = executable or config['exec']
mongo_params = params or config['params']
mongo_host = host or config['host']
mongo_port = get_port(port) or get_port(config['port'])
mongo_logsdir = logsdir or config['logsdir']
mongo_logpath = os.path.join(mongo_logsdir, f'mongo.{mongo_port}.log')
mongo_db_path = os.path.join(tmpdir, f'mongo.{mongo_port}')
os.mkdir(mongo_db_path)
request.addfinalizer(
lambda: os.path.exists(mongo_db_path) and rmtree(mongo_db_path))
mongo_executor = TCPExecutor(
(f'{mongo_exec} --bind_ip {mongo_host} --port {mongo_port} '
f'--dbpath {mongo_db_path} '
f'--logpath {mongo_logpath} {mongo_params}'),
host=mongo_host,
port=mongo_port,
timeout=60)
mongo_executor.start()
request.addfinalizer(mongo_executor.stop)
return mongo_executor
def test_fail_if_other_executor_running():
"""Test raising AlreadyRunning exception."""
executor = TCPExecutor(HTTP_SERVER, host='localhost', port=PORT)
executor2 = TCPExecutor(HTTP_SERVER, host='localhost', port=PORT)
with executor:
assert executor.running() is True
with pytest.raises(AlreadyRunning):
executor2.start()
with pytest.raises(AlreadyRunning):
with executor2:
pass
def test_start_and_wait():
"""Test if executor await for process to accept connections."""
command = 'bash -c "sleep 2 && nc -l 3000"'
executor = TCPExecutor(command, 'localhost', port=3000, timeout=5)
executor.start()
assert executor.running() is True
executor.stop()
# check proper __str__ and __repr__ rendering:
assert 'TCPExecutor' in repr(executor)
assert command in str(executor)
def memcached_server():
path = shutil.which('memcached')
if not path:
raise RuntimeError("Could not find memcached executable")
host = '127.0.0.1'
port = port_for.select_random()
executor = TCPExecutor(f'memcached -l {host} -p {port}', host, port)
executor.start()
yield executor
executor.stop()
executor.kill()
def local_matrix_server(use_matrix, use_local_matrix_server, local_matrix,
matrix_host, matrix_port):
if not (use_matrix and use_local_matrix_server):
yield None
return
assert local_matrix is not None, \
"No command to start the local matrix server given. (--local-matrix option)"
server = TCPExecutor(local_matrix,
host=matrix_host,
port=matrix_port,
timeout=120,
sleep=0.1,
shell=True)
server.start()
yield 'http://{}:{}'.format(matrix_host, matrix_port)
server.stop()
def dynamodb_proc_fixture(request):
"""
Process fixture for DynamoDB.
It starts DynamoDB when first used and stops it at the end
of the tests. Works on ``DynamoDBLocal.jar``.
:param FixtureRequest request: fixture request object
:rtype: pytest_dbfixtures.executors.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
path_dynamodb_jar = os.path.join((dynamodb_dir or config["dir"]),
"DynamoDBLocal.jar")
if not os.path.isfile(path_dynamodb_jar):
raise JarPathException(
"You have to provide a path to the dir with dynamodb jar file."
)
dynamodb_port = get_port(port or config["port"])
dynamodb_delay = ("-delayTransientStatuses"
if delay or config["delay"] else "")
dynamodb_host = host or config["host"]
dynamodb_executor = TCPExecutor(
f"""java
-Djava.library.path=./DynamoDBLocal_lib
-jar {path_dynamodb_jar}
-inMemory
{dynamodb_delay}
-port {dynamodb_port}""",
host=dynamodb_host,
port=dynamodb_port,
timeout=60,
)
dynamodb_executor.start()
yield dynamodb_executor
try:
dynamodb_executor.stop()
except ProcessExitedWithError:
pass
def local_matrix_server(transport_config):
if not transport_config.protocol == TransportProtocol.MATRIX:
yield None
return
assert transport_config.parameters.command is not None, \
'Missing --local-matrix setting. Cannot run Matrix version of integration test.'
server = TCPExecutor(
transport_config.parameters.command,
host=transport_config.parameters.host,
port=transport_config.parameters.port,
timeout=120,
sleep=0.1,
shell=True,
)
server.start()
yield f'http://{transport_config.parameters.host}:{transport_config.parameters.port}'
server.stop()
def mysql_proc_fixture(request):
"""
Process fixture for MySQL server.
#. Get config.
#. Initialize MySQL data directory
#. `Start a mysqld server
<https://dev.mysql.com/doc/refman/5.0/en/mysqld-safe.html>`_
#. Stop server and remove directory after tests.
`See <https://dev.mysql.com/doc/refman/5.6/en/mysqladmin.html>`_
:param FixtureRequest request: fixture request object
:rtype: pytest_dbfixtures.executors.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
mysql_exec = executable or config['exec']
mysql_admin_exec = admin_executable or config['admin']
mysql_init = init_executable or config['init']
mysql_port = get_port(port) or get_port(config['port'])
mysql_host = host or config['host']
mysql_params = params or config['params']
tmpdir = mkdtemp(prefix="pytest-mysql-")
datadir = os.path.join(
tmpdir,
'mysqldata_{port}'.format(port=mysql_port)
)
pidfile = os.path.join(
tmpdir,
'mysql-server.{port}.pid'.format(port=mysql_port)
)
unixsocket = os.path.join(
tmpdir,
'mysql.{port}.sock'.format(port=mysql_port)
)
logsdir = config['logsdir']
logfile_path = os.path.join(
logsdir,
'{prefix}mysql-server.{port}.log'.format(
prefix=logs_prefix,
port=mysql_port
)
)
init_mysql_directory(mysql_init, datadir, tmpdir)
mysql_executor = TCPExecutor(
'''
{mysql_server} --datadir={datadir} --pid-file={pidfile}
--port={port} --socket={socket} --log-error={logfile_path}
--tmpdir={tmpdir} --skip-syslog {params}
'''
.format(
mysql_server=mysql_exec,
port=mysql_port,
datadir=datadir,
pidfile=pidfile,
socket=unixsocket,
logfile_path=logfile_path,
params=mysql_params,
tmpdir=tmpdir,
),
host=mysql_host,
port=mysql_port,
timeout=60,
)
mysql_executor.socket_path = unixsocket
mysql_executor.start()
def stop_server_and_remove_directory():
shutdown_server = (
mysql_admin_exec,
'--socket=%s' % unixsocket,
'--user=%s' % config['user'],
'shutdown'
)
subprocess.check_output(' '.join(shutdown_server), shell=True)
mysql_executor.stop()
remove_mysql_directory(tmpdir)
request.addfinalizer(stop_server_and_remove_directory)
return mysql_executor
def test_repr_and_str():
"""Check the proper str and repr conversion."""
executor = TCPExecutor(NC_COMMAND, "localhost", port=3000, timeout=5)
# check proper __str__ and __repr__ rendering:
assert "TCPExecutor" in repr(executor)
assert NC_COMMAND in str(executor)
class LocalDynamoDB:
def __init__(self, path='/tmp/dynamodb'):
self._path = path
self._path_dynamodb_jar = os.path.join(path, 'DynamoDBLocal.jar')
self._port = self._get_open_port()
self.executor = TCPExecutor(
f'java -Djava.library.path=./DynamoDBLocal_lib -jar {self._path_dynamodb_jar} -inMemory -port {self._port}',
host='localhost',
port=self._port,
timeout=60,
)
# Write random credentials into env
self.aws_access_key = str(uuid4())
self.aws_secret_access_key = str(uuid4())
self.region = str(uuid4())
os.environ['AWS_ACCESS_KEY_ID'] = self.aws_access_key
os.environ['AWS_SECRET_ACCESS_KEY'] = self.aws_secret_access_key
os.environ['AWS_DEFAULT_REGION'] = self.region
self.__resources = set()
def start(self):
self._ensure_dynamodb_local()
self.executor.start()
return self
def __enter__(self):
self.start()
return self.resource()
def clear(self):
for t in self.resource().tables.all():
t.delete()
def stop(self):
# for resource in self.__resources:
# resource.
try:
self.executor.stop()
except ProcessFinishedWithError:
# Java exits with some strange code, ignore it, we wanted to stop it anyway
pass
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop()
def resource(self):
dynamo_db = boto3.resource(
'dynamodb',
endpoint_url='http://{host}:{port}'.format(
host='localhost',
port=self._port,
)
)
self.__resources.add(dynamo_db)
return dynamo_db
def credentials(self, table='table'):
return {
"access_key": self.aws_access_key,
"region": self.region,
"secret_access_key": self.aws_secret_access_key,
"table": table
}
@staticmethod
def _get_open_port():
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
s.listen(1)
port = s.getsockname()[1]
s.close()
return port
def _ensure_dynamodb_local(self):
if os.path.exists(self._path_dynamodb_jar):
print(f'Use existing DynamoDB setup in "{self._path}"')
else:
self._download_dynamodb()
def _download_dynamodb(self):
print(f'Download dynamodb local to "{self._path}"')
if os.path.exists(self._path):
print(f'Clean "{self._path}"')
shutil.rmtree(self._path)
with requests.get(LATEST_URL, stream=True) as r:
r.raise_for_status()
with tarfile.open(fileobj=r.raw, mode='r:gz') as tar:
tar.extractall(self._path)
for p in os.listdir(self._path):
print(p)
