class TestGRPCServer: def __init__(self, manage_py, params=None): if params is None: params = {} params.setdefault('--port', 50000 + randint(0, 10000)) self.port = params.get('--port') self.manage_py = manage_py self.process = TCPExecutor(['python', self.manage_py, 'grpcserver'] + list(self.flat_params(params)), host='localhost', port=self.port) @classmethod def flat_params(cls, dict_): for k, v in dict_.items(): yield str(k) if str(v) != "": yield str(v) def addr(self): return "localhost:%s" % self.port def start(self): self.process.start() def stop(self): self.process.stop()
def test_it_raises_error_on_timeout(): """Check if TimeoutExpired gets rised correctly.""" command = 'bash -c "sleep 10 && nc -l 3000"' executor = TCPExecutor(command, host='localhost', port=3000, timeout=2) with pytest.raises(TimeoutExpired): executor.start() assert executor.running() is False
def test_it_raises_error_on_timeout(): """Check if TimeoutExpired gets raised correctly.""" command = 'bash -c "sleep 10 && nc -l 3000"' executor = TCPExecutor(command, host='localhost', port=3000, timeout=2) with pytest.raises(TimeoutExpired): executor.start() assert executor.running() is False
def mongo_proc_fixture(request): """ Mongodb process fixture. :param FixtureRequest request: fixture request object :rtype: mirakuru.TCPExecutor :returns: tcp executor """ config = get_config(request) tmpdir = gettempdir() mongo_exec = executable or config['exec'] mongo_params = params or config['params'] mongo_host = host or config['host'] mongo_port = get_port(port) or get_port(config['port']) mongo_logsdir = logsdir or config['logsdir'] mongo_logpath = os.path.join( mongo_logsdir, 'mongo.{port}.log'.format( port=mongo_port ) ) mongo_db_path = os.path.join( tmpdir, 'mongo.{port}'.format( port=mongo_port ) ) os.mkdir(mongo_db_path) request.addfinalizer( lambda: os.path.exists(mongo_db_path) and rmtree(mongo_db_path) ) mongo_executor = TCPExecutor( ( '{mongo_exec} --bind_ip {host} --port {port}' ' --dbpath {dbpath} --logpath {logpath} {params}' ).format( mongo_exec=mongo_exec, params=mongo_params, host=mongo_host, port=mongo_port, dbpath=mongo_db_path, logpath=mongo_logpath, ), host=mongo_host, port=mongo_port, timeout=60 ) mongo_executor.start() request.addfinalizer(mongo_executor.stop) return mongo_executor
def __init__(self, manage_py, params=None): if params is None: params = {} params.setdefault('--port', 50000 + randint(0, 10000)) self.port = params.get('--port') self.manage_py = manage_py self.process = TCPExecutor(['python', self.manage_py, 'grpcserver'] + list(self.flat_params(params)), host='localhost', port=self.port)
def test_fail_if_other_executor_running(): """Test raising AlreadyRunning exception.""" executor = TCPExecutor(http_server_cmd, host='localhost', port=PORT) executor2 = TCPExecutor(http_server_cmd, host='localhost', port=PORT) with executor: assert executor.running() is True with pytest.raises(AlreadyRunning): executor2.start() with pytest.raises(AlreadyRunning): with executor2: pass
def test_default_port(): """ Test default port for the base TCP check. Check if HTTP executor fills in the default port for the TCP check from the base class if no port is provided in the URL. """ executor = HTTPExecutor(http_server_cmd, 'http://{0}/'.format(HOST)) assert executor.url.port is None assert executor.port == PORT assert TCPExecutor.pre_start_check(executor) is False executor.start() assert TCPExecutor.pre_start_check(executor) is True executor.stop()
def test_default_port(): """ Test default port for the base TCP check. Check if HTTP executor fills in the default port for the TCP check from the base class if no port is provided in the URL. """ executor = HTTPExecutor(http_server_cmd, 'http://{}/'.format(HOST)) assert executor.url.port is None assert executor.port == PORT assert TCPExecutor.pre_start_check(executor) is False executor.start() assert TCPExecutor.pre_start_check(executor) is True executor.stop()
def test_default_port(): """ Test default port for the base TCP check. Check if HTTP executor fills in the default port for the TCP check from the base class if no port is provided in the URL. """ executor = HTTPExecutor(HTTP_NORMAL_CMD, f'http://{HOST}/') assert executor.url.port is None assert executor.port == PORT assert TCPExecutor.pre_start_check(executor) is False executor.start() assert TCPExecutor.pre_start_check(executor) is True executor.stop()
def s3_client(monkeypatch, mocker, config): import botocore.session from mirakuru import TCPExecutor host = '127.0.0.1' socket, port = tornado.testing.bind_unused_port() socket.close() monkeypatch.setenv('TEST_SERVER_MODE', 'true') monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', '') monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'test-key') monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'test-secret-key') monkeypatch.setenv('AWS_SESSION_TOKEN', 'test-session-token') endpoint_url = "http://%s:%s" % (host, port) config.TC_AWS_ENDPOINT = endpoint_url mocker.patch('subprocess.Popen', side_effect=SubprocessStderrPipe) executor = TCPExecutor((sys.executable, '-m', 'moto.server', '-H', host, '-p', str(port), "s3"), host=host, port=port, timeout=10) try: with executor: yield botocore.session.get_session().create_client( 's3', endpoint_url=endpoint_url) finally: if asyncio is not None: event_loop = asyncio.get_event_loop() instances = getattr(Bucket, "_instances", None) or {} for bucket in instances.values(): event_loop.run_until_complete(bucket._client.close()) Bucket._instances = {}
def test_start_and_wait(caplog: LogCaptureFixture): """Test if executor await for process to accept connections.""" caplog.set_level(logging.DEBUG, logger="mirakuru") executor = TCPExecutor(NC_COMMAND, "localhost", port=3000, timeout=5) executor.start() assert executor.running() is True executor.stop()
def dynamodb_proc_fixture(request): """ Process fixture for DynamoDB. It starts DynamoDB when first used and stops it at the end of the tests. Works on ``DynamoDBLocal.jar``. :param FixtureRequest request: fixture request object :rtype: pytest_dbfixtures.executors.TCPExecutor :returns: tcp executor """ config = get_config(request) path_dynamodb_jar = os.path.join((dynamodb_dir or config['dir']), 'DynamoDBLocal.jar') if not os.path.isfile(path_dynamodb_jar): raise JarPathException( 'You have to provide a path to the dir with dynamodb jar file.' ) dynamodb_port = get_port(port or config['port']) dynamodb_delay = delay or config['delay'] dynamodb_host = host or config['host'] dynamodb_executor = TCPExecutor( '''java -Djava.library.path=./DynamoDBLocal_lib -jar {path_dynamodb_jar} -inMemory {delay} -port {port}'''.format( path_dynamodb_jar=path_dynamodb_jar, port=dynamodb_port, delay='-delayTransientStatuses' if dynamodb_delay else '', ), host=dynamodb_host, port=dynamodb_port, timeout=60, ) dynamodb_executor.start() request.addfinalizer(dynamodb_executor.stop) return dynamodb_executor
def oauth_redirect_server(temporary_path): config = temporary_path / 'config.yml' database = temporary_path / 'registered_clients' auth = uuid4().hex with config.open('w') as f: f.write('\n'.join( ('database: {}'.format(database), 'auth: {}'.format(auth)))) command = 'oauth-redirect -h 127.0.0.1 -p 8000 -a {} -d {}'.format( auth, database) server = TCPExecutor(command, host='127.0.0.1', port=8000, shell=True) server.start() server.config = config server.database = database server.auth = auth server.url = 'http://127.0.0.1:8000' yield server server.stop()
def moto_service(): executor = TCPExecutor( ('moto_server', '-H', host, '-p', str(port), service), host=host, port=port, timeout=timeout) connection_params.setdefault('proxy', host) connection_params.setdefault('proxy_port', port) connection_params.setdefault('aws_access_key_id', '') connection_params.setdefault('aws_secret_access_key', '') connection_params.setdefault('is_secure', False) with executor: yield get_connection(service, connection_params)
def __init__(self, path='/tmp/dynamodb'): self._path = path self._path_dynamodb_jar = os.path.join(path, 'DynamoDBLocal.jar') self._port = self._get_open_port() self.executor = TCPExecutor( f'java -Djava.library.path=./DynamoDBLocal_lib -jar {self._path_dynamodb_jar} -inMemory -port {self._port}', host='localhost', port=self._port, timeout=60, ) # Write random credentials into env self.aws_access_key = str(uuid4()) self.aws_secret_access_key = str(uuid4()) self.region = str(uuid4()) os.environ['AWS_ACCESS_KEY_ID'] = self.aws_access_key os.environ['AWS_SECRET_ACCESS_KEY'] = self.aws_secret_access_key os.environ['AWS_DEFAULT_REGION'] = self.region self.__resources = set()
def test_start_and_wait(timeout): """Test if executor await for process to accept connections.""" command = 'bash -c "sleep 2 && nc -l 3000"' executor = TCPExecutor(command, 'localhost', port=3000, timeout=timeout) executor.start() assert executor.running() is True executor.stop()
def test_start_and_wait(): """Test if executor await for process to accept connections.""" command = 'bash -c "sleep 2 && nc -l 3000"' executor = TCPExecutor(command, 'localhost', port=3000, timeout=5) executor.start() assert executor.running() is True executor.stop() # check proper __str__ and __repr__ rendering: assert 'TCPExecutor' in repr(executor) assert command in str(executor)
def memcached_server(): path = shutil.which('memcached') if not path: raise RuntimeError("Could not find memcached executable") host = '127.0.0.1' port = port_for.select_random() executor = TCPExecutor(f'memcached -l {host} -p {port}', host, port) executor.start() yield executor executor.stop() executor.kill()
def mongo_proc_fixture(request): """ Mongodb process fixture. :param FixtureRequest request: fixture request object :rtype: mirakuru.TCPExecutor :returns: tcp executor """ config = get_config(request) tmpdir = gettempdir() mongo_exec = executable or config['exec'] mongo_params = params or config['params'] mongo_host = host or config['host'] mongo_port = get_port(port) or get_port(config['port']) mongo_logsdir = logsdir or config['logsdir'] mongo_logpath = os.path.join(mongo_logsdir, f'mongo.{mongo_port}.log') mongo_db_path = os.path.join(tmpdir, f'mongo.{mongo_port}') os.mkdir(mongo_db_path) request.addfinalizer( lambda: os.path.exists(mongo_db_path) and rmtree(mongo_db_path)) mongo_executor = TCPExecutor( (f'{mongo_exec} --bind_ip {mongo_host} --port {mongo_port} ' f'--dbpath {mongo_db_path} ' f'--logpath {mongo_logpath} {mongo_params}'), host=mongo_host, port=mongo_port, timeout=60) mongo_executor.start() request.addfinalizer(mongo_executor.stop) return mongo_executor
def test_fail_if_other_executor_running(): """Test raising AlreadyRunning exception.""" executor = TCPExecutor(HTTP_SERVER, host='localhost', port=PORT) executor2 = TCPExecutor(HTTP_SERVER, host='localhost', port=PORT) with executor: assert executor.running() is True with pytest.raises(AlreadyRunning): executor2.start() with pytest.raises(AlreadyRunning): with executor2: pass
def local_matrix_server(use_matrix, use_local_matrix_server, local_matrix, matrix_host, matrix_port): if not (use_matrix and use_local_matrix_server): yield None return assert local_matrix is not None, \ "No command to start the local matrix server given. (--local-matrix option)" server = TCPExecutor(local_matrix, host=matrix_host, port=matrix_port, timeout=120, sleep=0.1, shell=True) server.start() yield 'http://{}:{}'.format(matrix_host, matrix_port) server.stop()
def dynamodb_proc_fixture(request): """ Process fixture for DynamoDB. It starts DynamoDB when first used and stops it at the end of the tests. Works on ``DynamoDBLocal.jar``. :param FixtureRequest request: fixture request object :rtype: pytest_dbfixtures.executors.TCPExecutor :returns: tcp executor """ config = get_config(request) path_dynamodb_jar = os.path.join((dynamodb_dir or config["dir"]), "DynamoDBLocal.jar") if not os.path.isfile(path_dynamodb_jar): raise JarPathException( "You have to provide a path to the dir with dynamodb jar file." ) dynamodb_port = get_port(port or config["port"]) dynamodb_delay = ("-delayTransientStatuses" if delay or config["delay"] else "") dynamodb_host = host or config["host"] dynamodb_executor = TCPExecutor( f"""java -Djava.library.path=./DynamoDBLocal_lib -jar {path_dynamodb_jar} -inMemory {dynamodb_delay} -port {dynamodb_port}""", host=dynamodb_host, port=dynamodb_port, timeout=60, ) dynamodb_executor.start() yield dynamodb_executor try: dynamodb_executor.stop() except ProcessExitedWithError: pass
def local_matrix_server(transport_config): if not transport_config.protocol == TransportProtocol.MATRIX: yield None return assert transport_config.parameters.command is not None, \ 'Missing --local-matrix setting. Cannot run Matrix version of integration test.' server = TCPExecutor( transport_config.parameters.command, host=transport_config.parameters.host, port=transport_config.parameters.port, timeout=120, sleep=0.1, shell=True, ) server.start() yield f'http://{transport_config.parameters.host}:{transport_config.parameters.port}' server.stop()
def mysql_proc_fixture(request): """ Process fixture for MySQL server. #. Get config. #. Initialize MySQL data directory #. `Start a mysqld server <https://dev.mysql.com/doc/refman/5.0/en/mysqld-safe.html>`_ #. Stop server and remove directory after tests. `See <https://dev.mysql.com/doc/refman/5.6/en/mysqladmin.html>`_ :param FixtureRequest request: fixture request object :rtype: pytest_dbfixtures.executors.TCPExecutor :returns: tcp executor """ config = get_config(request) mysql_exec = executable or config['exec'] mysql_admin_exec = admin_executable or config['admin'] mysql_init = init_executable or config['init'] mysql_port = get_port(port) or get_port(config['port']) mysql_host = host or config['host'] mysql_params = params or config['params'] tmpdir = mkdtemp(prefix="pytest-mysql-") datadir = os.path.join( tmpdir, 'mysqldata_{port}'.format(port=mysql_port) ) pidfile = os.path.join( tmpdir, 'mysql-server.{port}.pid'.format(port=mysql_port) ) unixsocket = os.path.join( tmpdir, 'mysql.{port}.sock'.format(port=mysql_port) ) logsdir = config['logsdir'] logfile_path = os.path.join( logsdir, '{prefix}mysql-server.{port}.log'.format( prefix=logs_prefix, port=mysql_port ) ) init_mysql_directory(mysql_init, datadir, tmpdir) mysql_executor = TCPExecutor( ''' {mysql_server} --datadir={datadir} --pid-file={pidfile} --port={port} --socket={socket} --log-error={logfile_path} --tmpdir={tmpdir} --skip-syslog {params} ''' .format( mysql_server=mysql_exec, port=mysql_port, datadir=datadir, pidfile=pidfile, socket=unixsocket, logfile_path=logfile_path, params=mysql_params, tmpdir=tmpdir, ), host=mysql_host, port=mysql_port, timeout=60, ) mysql_executor.socket_path = unixsocket mysql_executor.start() def stop_server_and_remove_directory(): shutdown_server = ( mysql_admin_exec, '--socket=%s' % unixsocket, '--user=%s' % config['user'], 'shutdown' ) subprocess.check_output(' '.join(shutdown_server), shell=True) mysql_executor.stop() remove_mysql_directory(tmpdir) request.addfinalizer(stop_server_and_remove_directory) return mysql_executor
def test_repr_and_str(): """Check the proper str and repr conversion.""" executor = TCPExecutor(NC_COMMAND, "localhost", port=3000, timeout=5) # check proper __str__ and __repr__ rendering: assert "TCPExecutor" in repr(executor) assert NC_COMMAND in str(executor)
class LocalDynamoDB: def __init__(self, path='/tmp/dynamodb'): self._path = path self._path_dynamodb_jar = os.path.join(path, 'DynamoDBLocal.jar') self._port = self._get_open_port() self.executor = TCPExecutor( f'java -Djava.library.path=./DynamoDBLocal_lib -jar {self._path_dynamodb_jar} -inMemory -port {self._port}', host='localhost', port=self._port, timeout=60, ) # Write random credentials into env self.aws_access_key = str(uuid4()) self.aws_secret_access_key = str(uuid4()) self.region = str(uuid4()) os.environ['AWS_ACCESS_KEY_ID'] = self.aws_access_key os.environ['AWS_SECRET_ACCESS_KEY'] = self.aws_secret_access_key os.environ['AWS_DEFAULT_REGION'] = self.region self.__resources = set() def start(self): self._ensure_dynamodb_local() self.executor.start() return self def __enter__(self): self.start() return self.resource() def clear(self): for t in self.resource().tables.all(): t.delete() def stop(self): # for resource in self.__resources: # resource. try: self.executor.stop() except ProcessFinishedWithError: # Java exits with some strange code, ignore it, we wanted to stop it anyway pass def __exit__(self, exc_type, exc_val, exc_tb): self.stop() def resource(self): dynamo_db = boto3.resource( 'dynamodb', endpoint_url='http://{host}:{port}'.format( host='localhost', port=self._port, ) ) self.__resources.add(dynamo_db) return dynamo_db def credentials(self, table='table'): return { "access_key": self.aws_access_key, "region": self.region, "secret_access_key": self.aws_secret_access_key, "table": table } @staticmethod def _get_open_port(): import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("", 0)) s.listen(1) port = s.getsockname()[1] s.close() return port def _ensure_dynamodb_local(self): if os.path.exists(self._path_dynamodb_jar): print(f'Use existing DynamoDB setup in "{self._path}"') else: self._download_dynamodb() def _download_dynamodb(self): print(f'Download dynamodb local to "{self._path}"') if os.path.exists(self._path): print(f'Clean "{self._path}"') shutil.rmtree(self._path) with requests.get(LATEST_URL, stream=True) as r: r.raise_for_status() with tarfile.open(fileobj=r.raw, mode='r:gz') as tar: tar.extractall(self._path) for p in os.listdir(self._path): print(p)