class TestGRPCServer:
def __init__(self, manage_py, params=None):
if params is None:
params = {}
params.setdefault('--port', 50000 + randint(0, 10000))
self.port = params.get('--port')
self.manage_py = manage_py
self.process = TCPExecutor(['python', self.manage_py, 'grpcserver'] +
list(self.flat_params(params)),
host='localhost',
port=self.port)
@classmethod
def flat_params(cls, dict_):
for k, v in dict_.items():
yield str(k)
if str(v) != "":
yield str(v)
def addr(self):
return "localhost:%s" % self.port
def start(self):
self.process.start()
def stop(self):
self.process.stop()
def test_start_and_wait(caplog: LogCaptureFixture):
"""Test if executor await for process to accept connections."""
caplog.set_level(logging.DEBUG, logger="mirakuru")
executor = TCPExecutor(NC_COMMAND, "localhost", port=3000, timeout=5)
executor.start()
assert executor.running() is True
executor.stop()
def test_start_and_wait(timeout):
"""Test if executor await for process to accept connections."""
command = 'bash -c "sleep 2 && nc -l 3000"'
executor = TCPExecutor(command, 'localhost', port=3000, timeout=timeout)
executor.start()
assert executor.running() is True
executor.stop()
def test_it_raises_error_on_timeout():
"""Check if TimeoutExpired gets raised correctly."""
command = 'bash -c "sleep 10 && nc -l 3000"'
executor = TCPExecutor(command, host='localhost', port=3000, timeout=2)
with pytest.raises(TimeoutExpired):
executor.start()
assert executor.running() is False
def test_it_raises_error_on_timeout():
"""Check if TimeoutExpired gets rised correctly."""
command = 'bash -c "sleep 10 && nc -l 3000"'
executor = TCPExecutor(command, host='localhost', port=3000, timeout=2)
with pytest.raises(TimeoutExpired):
executor.start()
assert executor.running() is False
def mongo_proc_fixture(request):
"""
Mongodb process fixture.
:param FixtureRequest request: fixture request object
:rtype: mirakuru.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
tmpdir = gettempdir()
mongo_exec = executable or config['exec']
mongo_params = params or config['params']
mongo_host = host or config['host']
mongo_port = get_port(port) or get_port(config['port'])
mongo_logsdir = logsdir or config['logsdir']
mongo_logpath = os.path.join(
mongo_logsdir,
'mongo.{port}.log'.format(
port=mongo_port
)
)
mongo_db_path = os.path.join(
tmpdir,
'mongo.{port}'.format(
port=mongo_port
)
)
os.mkdir(mongo_db_path)
request.addfinalizer(
lambda: os.path.exists(mongo_db_path) and rmtree(mongo_db_path)
)
mongo_executor = TCPExecutor(
(
'{mongo_exec} --bind_ip {host} --port {port}'
' --dbpath {dbpath} --logpath {logpath} {params}'
).format(
mongo_exec=mongo_exec,
params=mongo_params,
host=mongo_host,
port=mongo_port,
dbpath=mongo_db_path,
logpath=mongo_logpath,
),
host=mongo_host,
port=mongo_port,
timeout=60
)
mongo_executor.start()
request.addfinalizer(mongo_executor.stop)
return mongo_executor
def test_start_and_wait():
"""Test if executor await for process to accept connections."""
command = 'bash -c "sleep 2 && nc -l 3000"'
executor = TCPExecutor(command, 'localhost', port=3000, timeout=5)
executor.start()
assert executor.running() is True
executor.stop()
# check proper __str__ and __repr__ rendering:
assert 'TCPExecutor' in repr(executor)
assert command in str(executor)
def test_start_and_wait():
"""Test if executor await for process to accept connections."""
command = 'bash -c "sleep 2 && nc -l 3000"'
executor = TCPExecutor(command, 'localhost', port=3000, timeout=5)
executor.start()
assert executor.running() is True
executor.stop()
# check proper __str__ and __repr__ rendering:
assert 'TCPExecutor' in repr(executor)
assert command in str(executor)
def test_fail_if_other_executor_running():
"""Test raising AlreadyRunning exception."""
executor = TCPExecutor(http_server_cmd, host='localhost', port=PORT)
executor2 = TCPExecutor(http_server_cmd, host='localhost', port=PORT)
with executor:
assert executor.running() is True
with pytest.raises(AlreadyRunning):
executor2.start()
with pytest.raises(AlreadyRunning):
with executor2:
pass
def test_fail_if_other_executor_running():
"""Test raising AlreadyRunning exception."""
executor = TCPExecutor(HTTP_SERVER, host='localhost', port=PORT)
executor2 = TCPExecutor(HTTP_SERVER, host='localhost', port=PORT)
with executor:
assert executor.running() is True
with pytest.raises(AlreadyRunning):
executor2.start()
with pytest.raises(AlreadyRunning):
with executor2:
pass
def memcached_server():
path = shutil.which('memcached')
if not path:
raise RuntimeError("Could not find memcached executable")
host = '127.0.0.1'
port = port_for.select_random()
executor = TCPExecutor(f'memcached -l {host} -p {port}', host, port)
executor.start()
yield executor
executor.stop()
executor.kill()
def memcached_server():
path = shutil.which('memcached')
if not path:
raise RuntimeError("Could not find memcached executable")
host = '127.0.0.1'
port = port_for.select_random()
executor = TCPExecutor(f'memcached -l {host} -p {port}', host, port)
executor.start()
yield executor
executor.stop()
executor.kill()
def dynamodb_proc_fixture(request):
"""
Process fixture for DynamoDB.
It starts DynamoDB when first used and stops it at the end
of the tests. Works on ``DynamoDBLocal.jar``.
:param FixtureRequest request: fixture request object
:rtype: pytest_dbfixtures.executors.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
path_dynamodb_jar = os.path.join((dynamodb_dir or config['dir']),
'DynamoDBLocal.jar')
if not os.path.isfile(path_dynamodb_jar):
raise JarPathException(
'You have to provide a path to the dir with dynamodb jar file.'
)
dynamodb_port = get_port(port or config['port'])
dynamodb_delay = delay or config['delay']
dynamodb_host = host or config['host']
dynamodb_executor = TCPExecutor(
'''java
-Djava.library.path=./DynamoDBLocal_lib
-jar {path_dynamodb_jar}
-inMemory
{delay}
-port {port}'''.format(
path_dynamodb_jar=path_dynamodb_jar,
port=dynamodb_port,
delay='-delayTransientStatuses' if dynamodb_delay else '',
),
host=dynamodb_host,
port=dynamodb_port,
timeout=60,
)
dynamodb_executor.start()
request.addfinalizer(dynamodb_executor.stop)
return dynamodb_executor
def local_matrix_server(use_matrix, use_local_matrix_server, local_matrix,
matrix_host, matrix_port):
if not (use_matrix and use_local_matrix_server):
yield None
return
assert local_matrix is not None, \
"No command to start the local matrix server given. (--local-matrix option)"
server = TCPExecutor(local_matrix,
host=matrix_host,
port=matrix_port,
timeout=120,
sleep=0.1,
shell=True)
server.start()
yield 'http://{}:{}'.format(matrix_host, matrix_port)
server.stop()
def dynamodb_proc_fixture(request):
"""
Process fixture for DynamoDB.
It starts DynamoDB when first used and stops it at the end
of the tests. Works on ``DynamoDBLocal.jar``.
:param FixtureRequest request: fixture request object
:rtype: pytest_dbfixtures.executors.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
path_dynamodb_jar = os.path.join((dynamodb_dir or config["dir"]),
"DynamoDBLocal.jar")
if not os.path.isfile(path_dynamodb_jar):
raise JarPathException(
"You have to provide a path to the dir with dynamodb jar file."
)
dynamodb_port = get_port(port or config["port"])
dynamodb_delay = ("-delayTransientStatuses"
if delay or config["delay"] else "")
dynamodb_host = host or config["host"]
dynamodb_executor = TCPExecutor(
f"""java
-Djava.library.path=./DynamoDBLocal_lib
-jar {path_dynamodb_jar}
-inMemory
{dynamodb_delay}
-port {dynamodb_port}""",
host=dynamodb_host,
port=dynamodb_port,
timeout=60,
)
dynamodb_executor.start()
yield dynamodb_executor
try:
dynamodb_executor.stop()
except ProcessExitedWithError:
pass
def local_matrix_server(transport_config):
if not transport_config.protocol == TransportProtocol.MATRIX:
yield None
return
assert transport_config.parameters.command is not None, \
'Missing --local-matrix setting. Cannot run Matrix version of integration test.'
server = TCPExecutor(
transport_config.parameters.command,
host=transport_config.parameters.host,
port=transport_config.parameters.port,
timeout=120,
sleep=0.1,
shell=True,
)
server.start()
yield f'http://{transport_config.parameters.host}:{transport_config.parameters.port}'
server.stop()
def oauth_redirect_server(temporary_path):
config = temporary_path / 'config.yml'
database = temporary_path / 'registered_clients'
auth = uuid4().hex
with config.open('w') as f:
f.write('\n'.join(
('database: {}'.format(database), 'auth: {}'.format(auth))))
command = 'oauth-redirect -h 127.0.0.1 -p 8000 -a {} -d {}'.format(
auth, database)
server = TCPExecutor(command, host='127.0.0.1', port=8000, shell=True)
server.start()
server.config = config
server.database = database
server.auth = auth
server.url = 'http://127.0.0.1:8000'
yield server
server.stop()
def mongo_proc_fixture(request):
"""
Mongodb process fixture.
:param FixtureRequest request: fixture request object
:rtype: mirakuru.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
tmpdir = gettempdir()
mongo_exec = executable or config['exec']
mongo_params = params or config['params']
mongo_host = host or config['host']
mongo_port = get_port(port) or get_port(config['port'])
mongo_logsdir = logsdir or config['logsdir']
mongo_logpath = os.path.join(mongo_logsdir, f'mongo.{mongo_port}.log')
mongo_db_path = os.path.join(tmpdir, f'mongo.{mongo_port}')
os.mkdir(mongo_db_path)
request.addfinalizer(
lambda: os.path.exists(mongo_db_path) and rmtree(mongo_db_path))
mongo_executor = TCPExecutor(
(f'{mongo_exec} --bind_ip {mongo_host} --port {mongo_port} '
f'--dbpath {mongo_db_path} '
f'--logpath {mongo_logpath} {mongo_params}'),
host=mongo_host,
port=mongo_port,
timeout=60)
mongo_executor.start()
request.addfinalizer(mongo_executor.stop)
return mongo_executor
def mysql_proc_fixture(request):
"""
Process fixture for MySQL server.
#. Get config.
#. Initialize MySQL data directory
#. `Start a mysqld server
<https://dev.mysql.com/doc/refman/5.0/en/mysqld-safe.html>`_
#. Stop server and remove directory after tests.
`See <https://dev.mysql.com/doc/refman/5.6/en/mysqladmin.html>`_
:param FixtureRequest request: fixture request object
:rtype: pytest_dbfixtures.executors.TCPExecutor
:returns: tcp executor
"""
config = get_config(request)
mysql_exec = executable or config['exec']
mysql_admin_exec = admin_executable or config['admin']
mysql_init = init_executable or config['init']
mysql_port = get_port(port) or get_port(config['port'])
mysql_host = host or config['host']
mysql_params = params or config['params']
tmpdir = mkdtemp(prefix="pytest-mysql-")
datadir = os.path.join(
tmpdir,
'mysqldata_{port}'.format(port=mysql_port)
)
pidfile = os.path.join(
tmpdir,
'mysql-server.{port}.pid'.format(port=mysql_port)
)
unixsocket = os.path.join(
tmpdir,
'mysql.{port}.sock'.format(port=mysql_port)
)
logsdir = config['logsdir']
logfile_path = os.path.join(
logsdir,
'{prefix}mysql-server.{port}.log'.format(
prefix=logs_prefix,
port=mysql_port
)
)
init_mysql_directory(mysql_init, datadir, tmpdir)
mysql_executor = TCPExecutor(
'''
{mysql_server} --datadir={datadir} --pid-file={pidfile}
--port={port} --socket={socket} --log-error={logfile_path}
--tmpdir={tmpdir} --skip-syslog {params}
'''
.format(
mysql_server=mysql_exec,
port=mysql_port,
datadir=datadir,
pidfile=pidfile,
socket=unixsocket,
logfile_path=logfile_path,
params=mysql_params,
tmpdir=tmpdir,
),
host=mysql_host,
port=mysql_port,
timeout=60,
)
mysql_executor.socket_path = unixsocket
mysql_executor.start()
def stop_server_and_remove_directory():
shutdown_server = (
mysql_admin_exec,
'--socket=%s' % unixsocket,
'--user=%s' % config['user'],
'shutdown'
)
subprocess.check_output(' '.join(shutdown_server), shell=True)
mysql_executor.stop()
remove_mysql_directory(tmpdir)
request.addfinalizer(stop_server_and_remove_directory)
return mysql_executor
class LocalDynamoDB:
def __init__(self, path='/tmp/dynamodb'):
self._path = path
self._path_dynamodb_jar = os.path.join(path, 'DynamoDBLocal.jar')
self._port = self._get_open_port()
self.executor = TCPExecutor(
f'java -Djava.library.path=./DynamoDBLocal_lib -jar {self._path_dynamodb_jar} -inMemory -port {self._port}',
host='localhost',
port=self._port,
timeout=60,
)
# Write random credentials into env
self.aws_access_key = str(uuid4())
self.aws_secret_access_key = str(uuid4())
self.region = str(uuid4())
os.environ['AWS_ACCESS_KEY_ID'] = self.aws_access_key
os.environ['AWS_SECRET_ACCESS_KEY'] = self.aws_secret_access_key
os.environ['AWS_DEFAULT_REGION'] = self.region
self.__resources = set()
def start(self):
self._ensure_dynamodb_local()
self.executor.start()
return self
def __enter__(self):
self.start()
return self.resource()
def clear(self):
for t in self.resource().tables.all():
t.delete()
def stop(self):
# for resource in self.__resources:
# resource.
try:
self.executor.stop()
except ProcessFinishedWithError:
# Java exits with some strange code, ignore it, we wanted to stop it anyway
pass
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop()
def resource(self):
dynamo_db = boto3.resource(
'dynamodb',
endpoint_url='http://{host}:{port}'.format(
host='localhost',
port=self._port,
)
)
self.__resources.add(dynamo_db)
return dynamo_db
def credentials(self, table='table'):
return {
"access_key": self.aws_access_key,
"region": self.region,
"secret_access_key": self.aws_secret_access_key,
"table": table
}
@staticmethod
def _get_open_port():
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
s.listen(1)
port = s.getsockname()[1]
s.close()
return port
def _ensure_dynamodb_local(self):
if os.path.exists(self._path_dynamodb_jar):
print(f'Use existing DynamoDB setup in "{self._path}"')
else:
self._download_dynamodb()
def _download_dynamodb(self):
print(f'Download dynamodb local to "{self._path}"')
if os.path.exists(self._path):
print(f'Clean "{self._path}"')
shutil.rmtree(self._path)
with requests.get(LATEST_URL, stream=True) as r:
r.raise_for_status()
with tarfile.open(fileobj=r.raw, mode='r:gz') as tar:
tar.extractall(self._path)
for p in os.listdir(self._path):
print(p)