def local_matrix_server(transport_config): if not transport_config.protocol == TransportProtocol.MATRIX: yield None return server = transport_config.parameters.server # if command is none, assume server is already running if transport_config.parameters.command in (None, 'none'): yield server return # otherwise, run our own local server matrix = HTTPExecutor( transport_config.parameters.command, status=r'^[24]\d\d$', url=urljoin(server, '/_matrix'), timeout=120, sleep=0.1, shell=True, ) matrix.start() yield server matrix.stop()
def elasticsearch_proc_fixture(request): """Elasticsearch process starting fixture.""" elasticsearch_port = get_port(port) pidfile = '/tmp/elasticsearch.{0}.pid'.format(elasticsearch_port) home_path = '/tmp/elasticsearch_{0}'.format(elasticsearch_port) logsdir = path(request.config.getvalue('logsdir')) logs_path = logsdir / '{prefix}elasticsearch_{port}_logs'.format( prefix=logs_prefix, port=elasticsearch_port ) work_path = '/tmp/elasticsearch_{0}_tmp'.format(elasticsearch_port) cluster = cluster_name or 'dbfixtures.{0}'.format(elasticsearch_port) multicast_enabled = str(discovery_zen_ping_multicast_enabled).lower() command_exec = ''' {deamon} -p {pidfile} --http.port={port} --path.home={home_path} --default.path.logs={logs_path} --default.path.work={work_path} --default.path.conf=/etc/elasticsearch --cluster.name={cluster} --network.publish_host='{network_publish_host}' --discovery.zen.ping.multicast.enabled={multicast_enabled} --index.store.type={index_store_type} '''.format( deamon=executable, pidfile=pidfile, port=elasticsearch_port, home_path=home_path, logs_path=logs_path, work_path=work_path, cluster=cluster, network_publish_host=network_publish_host, multicast_enabled=multicast_enabled, index_store_type=index_store_type ) elasticsearch_executor = HTTPExecutor( command_exec, 'http://{host}:{port}'.format( host=host, port=elasticsearch_port ), timeout=60, ) elasticsearch_executor.start() def finalize_elasticsearch(): elasticsearch_executor.stop() shutil.rmtree(home_path) request.addfinalizer(finalize_elasticsearch) return elasticsearch_executor
def backend(request): """Run application backend.""" executor = HTTPExecutor( [ 'planningpoker', '--host', '127.0.0.1', '--port', str(PORT), '--cookie-secret-key', Fernet.generate_key().decode() ], SITE_ADDRESS + '/status', timeout=EXECUTOR_TIMEOUT ) executor.start() request.addfinalizer(executor.stop)
def test_shell_started_server_stops(): """Test if executor terminates properly executor with shell=True.""" executor = HTTPExecutor(http_server_cmd, "http://{0}:{1}/".format(HOST, PORT), timeout=20, shell=True) with pytest.raises(socket.error): connect_to_server() with executor: assert executor.running() is True connect_to_server() assert executor.running() is False with pytest.raises(socket.error): connect_to_server()
def test_slow_method_server_timed_out(method): """Check if timeout properly expires.""" http_method_slow_cmd = ( f'{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT} False {method}') executor = HTTPExecutor(http_method_slow_cmd, f'http://{HOST}:{PORT}/', method=method, timeout=1) with pytest.raises(TimeoutExpired) as exc: executor.start() assert executor.running() is False assert 'timed out after' in str(exc.value)
def test_default_port(): """ Test default port for the base TCP check. Check if HTTP executor fills in the default port for the TCP check from the base class if no port is provided in the URL. """ executor = HTTPExecutor(http_server_cmd, 'http://{}/'.format(HOST)) assert executor.url.port is None assert executor.port == PORT assert TCPExecutor.pre_start_check(executor) is False executor.start() assert TCPExecutor.pre_start_check(executor) is True executor.stop()
def test_fail_if_other_executor_running(): """Test raising AlreadyRunning exception when port is blocked.""" executor = HTTPExecutor(http_server_cmd, "http://{0}:{1}/".format(HOST, PORT)) executor2 = HTTPExecutor(http_server_cmd, "http://{0}:{1}/".format(HOST, PORT)) with executor: assert executor.running() is True with pytest.raises(AlreadyRunning): executor2.start() with pytest.raises(AlreadyRunning) as exc: with executor2: pass assert "seems to be already running" in str(exc)
def test_shell_started_server_stops(): """Test if executor terminates properly executor with shell=True.""" executor = HTTPExecutor(http_server_cmd, 'http://{0}:{1}/'.format(HOST, PORT), timeout=20, shell=True) with pytest.raises(socket.error): connect_to_server() with executor: assert executor.running() is True connect_to_server() assert executor.running() is False with pytest.raises(socket.error): connect_to_server()
def test_shell_started_server_stops(): """Test if executor terminates properly executor with shell=True.""" executor = HTTPExecutor(HTTP_NORMAL_CMD, f'http://{HOST}:{PORT}/', timeout=20, shell=True) with pytest.raises(socket.error): connect_to_server() with executor: assert executor.running() is True connect_to_server() assert executor.running() is False with pytest.raises(socket.error): connect_to_server()
def test_fail_if_other_executor_running(): """Test raising AlreadyRunning exception when port is blocked.""" executor = HTTPExecutor( http_server_cmd, 'http://{0}:{1}/'.format(HOST, PORT), ) executor2 = HTTPExecutor( http_server_cmd, 'http://{0}:{1}/'.format(HOST, PORT), ) with executor: assert executor.running() is True with pytest.raises(AlreadyRunning): executor2.start() with pytest.raises(AlreadyRunning): with executor2: pass
def test_http_status_codes(accepted_status, expected_timeout): """ Test how 'status' argument influences executor start. :param int|str accepted_status: Executor 'status' value :param bool expected_timeout: if Executor raises TimeoutExpired or not """ kwargs = { 'command': http_server_cmd, 'url': 'http://{0}:{1}/badpath'.format(HOST, PORT), 'timeout': 2 } if accepted_status: kwargs['status'] = accepted_status executor = HTTPExecutor(**kwargs) if not expected_timeout: executor.start() executor.stop() else: with pytest.raises(TimeoutExpired): executor.start() executor.stop()
def test_http_status_codes(accepted_status, expected_timeout): """ Test how 'status' argument influences executor start. :param int|str accepted_status: Executor 'status' value :param bool expected_timeout: if Executor raises TimeoutExpired or not """ kwargs: Dict[str, Any] = { 'command': HTTP_NORMAL_CMD, 'url': f'http://{HOST}:{PORT}/badpath', 'timeout': 2 } if accepted_status: kwargs['status'] = accepted_status executor = HTTPExecutor(**kwargs) if not expected_timeout: executor.start() executor.stop() else: with pytest.raises(TimeoutExpired): executor.start() executor.stop()
def test_stopping_brutally(): """ Test if SimpleExecutor is stopping insubordinate process. Check if the process that doesn't react to SIGTERM signal will be killed by executor with SIGKILL automatically. """ host_port = "127.0.0.1:8000" cmd = f"{sys.executable} {TEST_SERVER_PATH} {host_port} True" executor = HTTPExecutor(cmd, f"http://{host_port!s}/", timeout=20) executor.start() assert executor.running() is True stop_at = time.time() + 10 executor.stop() assert executor.running() is False assert stop_at <= time.time(), "Subprocess killed earlier than in 10 secs"
def test_http_status_codes(accepted_status: Union[None, int, str], expected_timeout: bool) -> None: """ Test how 'status' argument influences executor start. :param int|str accepted_status: Executor 'status' value :param bool expected_timeout: if Executor raises TimeoutExpired or not """ kwargs: Dict[str, Any] = { "command": HTTP_NORMAL_CMD, "url": f"http://{HOST}:{PORT}/badpath", "timeout": 2, } if accepted_status: kwargs["status"] = accepted_status executor = HTTPExecutor(**kwargs) if not expected_timeout: executor.start() executor.stop() else: with pytest.raises(TimeoutExpired): executor.start() executor.stop()
def test_stopping_brutally(): """ Test if Executor is stopping insubordinate process. Check if the process that doesn't react to SIGTERM signal will be killed by executor with SIGKILL automatically. """ host_port = "127.0.0.1:8000" cmd = '{} {} {} True'.format(sys.executable, test_server_path, host_port) executor = HTTPExecutor(cmd, 'http://%s/' % host_port) executor.start() assert executor.running() is True stop_at = time.time() + 10 executor.stop() assert executor.running() is False assert stop_at <= time.time(), "Subprocess killed earlier than in 10 secs"
def test_slow_method_server_starting(method): """ Test whether or not executor awaits for slow starting servers. Simple example. You run Gunicorn and it is working but you have to wait for worker processes. """ http_method_slow_cmd = ( f'{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT} False {method}') with HTTPExecutor(http_method_slow_cmd, f'http://{HOST}:{PORT}/', method=method, timeout=30) as executor: assert executor.running() is True connect_to_server()
def test_executor_starts_and_waits(): """Test if process awaits for HEAD request to be completed.""" command = f'bash -c "sleep 3 && {HTTP_NORMAL_CMD}"' executor = HTTPExecutor(command, f'http://{HOST}:{PORT}/', timeout=20) executor.start() assert executor.running() is True connect_to_server() executor.stop() # check proper __str__ and __repr__ rendering: assert 'HTTPExecutor' in repr(executor) assert command in str(executor)
def test_executor_starts_and_waits(): """Test if process awaits for HEAD request to be completed.""" command = 'bash -c "sleep 3 && {0}"'.format(http_server_cmd) executor = HTTPExecutor(command, 'http://{0}:{1}/'.format(HOST, PORT), timeout=20) executor.start() assert executor.running() is True connect_to_server() executor.stop() # check proper __str__ and __repr__ rendering: assert 'HTTPExecutor' in repr(executor) assert command in str(executor)
def test_stopping_brutally(): """ Test if SimpleExecutor is stopping insubordinate process. Check if the process that doesn't react to SIGTERM signal will be killed by executor with SIGKILL automatically. """ host_port = "127.0.0.1:8000" cmd = '{} {} {} True'.format(sys.executable, test_server_path, host_port) executor = HTTPExecutor(cmd, 'http://%s/' % host_port) executor.start() assert executor.running() is True stop_at = time.time() + 10 executor.stop() assert executor.running() is False assert stop_at <= time.time(), "Subprocess killed earlier than in 10 secs"
def test_slow_post_payload_server_starting(): """ Test whether or not executor awaits for slow starting servers. Simple example. You run Gunicorn and it is working but you have to wait for worker processes. """ http_method_slow_cmd = ( f"{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT} False Key") with HTTPExecutor( http_method_slow_cmd, f"http://{HOST}:{PORT}/", method="POST", timeout=30, payload={"key": "hole"}, ) as executor: assert executor.running() is True connect_to_server()
def test_executor_starts_and_waits(): """Test if process awaits for HEAD request to be completed.""" command = 'bash -c "sleep 3 && {0}"'.format(http_server_cmd) executor = HTTPExecutor( command, 'http://{0}:{1}/'.format(HOST, PORT), timeout=20 ) executor.start() assert executor.running() is True connect_to_server() executor.stop()
def test_executor_starts_and_waits(): """Test if process awaits for HEAD request to be completed.""" command = 'bash -c "sleep 3 && {0}"'.format(http_server_cmd) executor = HTTPExecutor(command, "http://{0}:{1}/".format(HOST, PORT), timeout=20) executor.start() assert executor.running() is True connect_to_server() executor.stop() # check proper __str__ and __repr__ rendering: assert "HTTPExecutor" in repr(executor) assert command in str(executor)
def test_default_port(): """ Test default port for the base TCP check. Check if HTTP executor fills in the default port for the TCP check from the base class if no port is provided in the URL. """ executor = HTTPExecutor(http_server_cmd, 'http://{0}/'.format(HOST)) assert executor.url.port is None assert executor.port == PORT assert TCPExecutor.pre_start_check(executor) is False executor.start() assert TCPExecutor.pre_start_check(executor) is True executor.stop()
def test_default_port(): """ Test default port for the base TCP check. Check if HTTP executor fills in the default port for the TCP check from the base class if no port is provided in the URL. """ executor = HTTPExecutor(HTTP_NORMAL_CMD, f'http://{HOST}/') assert executor.url.port is None assert executor.port == PORT assert TCPExecutor.pre_start_check(executor) is False executor.start() assert TCPExecutor.pre_start_check(executor) is True executor.stop()
def es_process(es_binary, es_version): port = port_for.select_random() pid = es_binary + '.pid' # use a different garbage collector for better performance os.environ['ES_JAVA_OPTS'] = \ '-Xms1g -Xmx1g -XX:-UseConcMarkSweepGC -XX:+UseG1GC' command = ( f"{es_binary} -p {pid} -E http.port={port} " f"-E xpack.monitoring.enabled=false " f"-E xpack.monitoring.collection.enabled=false " f"> /dev/null" ) url = f'http://127.0.0.1:{port}/_cluster/health?wait_for_status=green' executor = HTTPExecutor(command, url, method='GET', shell=True) executor.start() yield executor executor.stop() executor.kill()
def test_fail_if_other_executor_running(): """Test raising AlreadyRunning exception when port is blocked.""" executor = HTTPExecutor( http_server_cmd, 'http://{0}:{1}/'.format(HOST, PORT), ) executor2 = HTTPExecutor( http_server_cmd, 'http://{0}:{1}/'.format(HOST, PORT), ) with executor: assert executor.running() is True with pytest.raises(AlreadyRunning): executor2.start() with pytest.raises(AlreadyRunning) as exc: with executor2: pass assert 'seems to be already running' in str(exc)
def test_fail_if_other_running(): """Test raising AlreadyRunning exception when port is blocked.""" executor = HTTPExecutor( HTTP_NORMAL_CMD, f'http://{HOST}:{PORT}/', ) executor2 = HTTPExecutor( HTTP_NORMAL_CMD, f'http://{HOST}:{PORT}/', ) with executor: assert executor.running() is True with pytest.raises(AlreadyRunning): executor2.start() with pytest.raises(AlreadyRunning) as exc: with executor2: pass assert 'seems to be already running' in str(exc.value)
def elasticsearch_proc_fixture(request): """Elasticsearch process starting fixture.""" tmpdir = gettempdir() config = return_config(request) elasticsearch_host = host or config["host"] elasticsearch_port = get_port(port) or get_port(config["port"]) elasticsearch_cluster_name = ( cluster_name or config["cluster_name"] or "elasticsearch_cluster_{0}".format(elasticsearch_port) ) elasticsearch_logs_prefix = logs_prefix or config["logs_prefix"] elasticsearch_index_store_type = index_store_type or config["index_store_type"] elasticsearch_network_publish_host = network_publish_host or config["network_publish_host"] logsdir = elasticsearch_logsdir or config["logsdir"] logs_path = os.path.join( logsdir, "{prefix}elasticsearch_{port}_logs".format(prefix=elasticsearch_logs_prefix, port=elasticsearch_port), ) pidfile = os.path.join(tmpdir, "elasticsearch.{0}.pid".format(elasticsearch_port)) home_path = os.path.join(tmpdir, "elasticsearch_{0}".format(elasticsearch_port)) work_path = "{0}_tmp".format(home_path) if discovery_zen_ping_multicast_enabled is not None: multicast_enabled = str(discovery_zen_ping_multicast_enabled).lower() else: multicast_enabled = config["discovery_zen_ping_multicast_enabled"] command_exec = """ {deamon} -p {pidfile} --http.port={port} --path.home={home_path} --default.path.logs={logs_path} --default.path.work={work_path} --default.path.conf=/etc/elasticsearch --cluster.name={cluster} --network.publish_host='{network_publish_host}' --discovery.zen.ping.multicast.enabled={multicast_enabled} --index.store.type={index_store_type} """.format( deamon=executable, pidfile=pidfile, port=elasticsearch_port, home_path=home_path, logs_path=logs_path, work_path=work_path, cluster=elasticsearch_cluster_name, network_publish_host=elasticsearch_network_publish_host, multicast_enabled=multicast_enabled, index_store_type=elasticsearch_index_store_type, ) elasticsearch_executor = HTTPExecutor( command_exec, "http://{host}:{port}".format(host=elasticsearch_host, port=elasticsearch_port), timeout=60 ) elasticsearch_executor.start() def finalize_elasticsearch(): elasticsearch_executor.stop() shutil.rmtree(home_path) request.addfinalizer(finalize_elasticsearch) return elasticsearch_executor
def smoketest(ctx, debug, local_matrix, **kwargs): # pylint: disable=unused-argument """ Test, that the raiden installation is sane. """ import binascii from raiden.api.python import RaidenAPI from raiden.blockchain.abi import get_static_or_compile from raiden.tests.utils.blockchain import geth_wait_and_check from raiden.tests.integration.fixtures.backend_geth import web3 from raiden.tests.utils.smoketest import ( load_smoketest_config, start_ethereum, run_smoketests, patch_smoke_fns, ) from raiden.utils import get_contract_path from raiden.raiden_service import RaidenService # TODO: Temporary until we also deploy contracts in smoketests # This function call patches the initial query filtering to create some fake # events. That is done to make up for the missing events that would have # been generated and populated the state if the contracts were deployed # and not precompiled in the smoketest genesis file RaidenService.install_and_query_payment_network_filters = patch_smoke_fns( RaidenService.install_and_query_payment_network_filters, ) # Check the solidity compiler early in the smoketest. # # Binary distributions don't need the solidity compiler but source # distributions do. Since this is checked by `get_static_or_compile` # function, use it as a proxy for validating the setup. get_static_or_compile( get_contract_path('HumanStandardToken.sol'), 'HumanStandardToken', ) report_file = tempfile.mktemp(suffix='.log') configure_logging({'': 'DEBUG'}, log_file=report_file) def append_report(subject, data): with open(report_file, 'a', encoding='UTF-8') as handler: handler.write(f'{f" {subject.upper()} ":=^80}{os.linesep}') if data is not None: if isinstance(data, bytes): data = data.decode() handler.writelines([data + os.linesep]) append_report('raiden version', json.dumps(get_system_spec())) append_report('raiden log', None) print('[1/5] getting smoketest configuration') smoketest_config = load_smoketest_config() if not smoketest_config: append_report( 'smoketest configuration', 'Could not load the smoketest genesis configuration file.', ) print('[2/5] starting ethereum') ethereum, ethereum_config = start_ethereum(smoketest_config['genesis']) port = ethereum_config['rpc'] web3_client = web3([port]) random_marker = binascii.hexlify(b'raiden').decode() privatekeys = [] geth_wait_and_check(web3_client, privatekeys, random_marker) print('[3/5] starting raiden') # setup cli arguments for starting raiden args = dict( discovery_contract_address=smoketest_config['contracts'] ['discovery_address'], registry_contract_address=smoketest_config['contracts'] ['registry_address'], secret_registry_contract_address=smoketest_config['contracts'] ['secret_registry_address'], eth_rpc_endpoint='http://127.0.0.1:{}'.format(port), keystore_path=ethereum_config['keystore'], address=ethereum_config['address'], network_id='627', transport=ctx.parent.params['transport'], matrix_server='http://localhost:8008' if ctx.parent.params['matrix_server'] == 'auto' else ctx.parent.params['matrix_server'], ) smoketest_config['transport'] = args['transport'] for option_ in app.params: if option_.name in args.keys(): args[option_.name] = option_.process_value(ctx, args[option_.name]) else: args[option_.name] = option_.default password_file = os.path.join(args['keystore_path'], 'password') with open(password_file, 'w') as handler: handler.write('password') port = next(get_free_port('127.0.0.1', 5001)) args['password_file'] = click.File()(password_file) args['datadir'] = args['keystore_path'] args['api_address'] = 'localhost:' + str(port) args['sync_check'] = False def _run_smoketest(): # invoke the raiden app app_ = ctx.invoke(app, **args) raiden_api = RaidenAPI(app_.raiden) rest_api = RestAPI(raiden_api) api_server = APIServer(rest_api) (api_host, api_port) = split_endpoint(args['api_address']) api_server.start(api_host, api_port) success = False try: print('[4/5] running smoketests...') error = run_smoketests(app_.raiden, smoketest_config, debug=debug) if error is not None: append_report('smoketest assertion error', error) else: success = True finally: app_.stop() ethereum.send_signal(2) err, out = ethereum.communicate() append_report('geth init stdout', ethereum_config['init_log_out'].decode('utf-8')) append_report('geth init stderr', ethereum_config['init_log_err'].decode('utf-8')) append_report('ethereum stdout', out) append_report('ethereum stderr', err) append_report('smoketest configuration', json.dumps(smoketest_config)) if success: print( '[5/5] smoketest successful, report was written to {}'.format( report_file)) else: print( '[5/5] smoketest had errors, report was written to {}'.format( report_file)) return success if args['transport'] == 'udp': with SocketFactory('127.0.0.1', port, strategy='none') as mapped_socket: args['mapped_socket'] = mapped_socket success = _run_smoketest() elif args['transport'] == 'matrix' and local_matrix.lower() != 'none': print('WARNING: The Matrix transport is experimental') args['mapped_socket'] = None with HTTPExecutor( local_matrix, status=r'^[24]\d\d$', url=urljoin(args['matrix_server'], '/_matrix'), ): args['extra_config'] = { 'matrix': { 'discovery_room': { 'server': 'matrix.local.raiden' }, 'server_name': 'matrix.local.raiden', }, } success = _run_smoketest() elif args['transport'] == 'matrix' and local_matrix.lower() == "none": print('WARNING: The Matrix transport is experimental') args['mapped_socket'] = None success = _run_smoketest() else: # Shouldn't happen raise RuntimeError(f"Invalid transport type '{args['transport']}'") if not success: sys.exit(1)
def elasticsearch_proc_fixture(request): """Elasticsearch process starting fixture.""" config = return_config(request) elasticsearch_host = host or config['host'] elasticsearch_port = get_port(port) or get_port(config['port']) elasticsearch_transport_port = get_port(transport_tcp_port) or \ get_port(config['transport_tcp_port']) elasticsearch_cluster_name = \ cluster_name or config['cluster_name'] or \ 'elasticsearch_cluster_{0}'.format(elasticsearch_port) elasticsearch_logs_prefix = logs_prefix or config['logs_prefix'] elasticsearch_index_store_type = index_store_type or \ config['index_store_type'] elasticsearch_network_publish_host = network_publish_host or \ config['network_publish_host'] logsdir = elasticsearch_logsdir or config['logsdir'] logs_path = os.path.join( logsdir, '{prefix}elasticsearch_{port}_logs'.format( prefix=elasticsearch_logs_prefix, port=elasticsearch_port )) pidfile = os.path.join( gettempdir(), 'elasticsearch.{0}.pid'.format(elasticsearch_port)) home_path = os.path.join( gettempdir(), 'elasticsearch_{0}'.format(elasticsearch_port)) work_path = '{0}_tmp'.format(home_path) conf_path = configuration_path or config['configuration_path'] if discovery_zen_ping_multicast_enabled is not None: multicast_enabled = str( discovery_zen_ping_multicast_enabled).lower() else: multicast_enabled = config['discovery_zen_ping_multicast_enabled'] command = command_from(version=get_version_parts(executable)) command_exec = command.format( deamon=executable, pidfile=pidfile, port=elasticsearch_port, transport_tcp_port=elasticsearch_transport_port, conf_path=conf_path, home_path=home_path, logs_path=logs_path, work_path=work_path, cluster=elasticsearch_cluster_name, network_publish_host=elasticsearch_network_publish_host, multicast_enabled=multicast_enabled, index_store_type=elasticsearch_index_store_type ) elasticsearch_executor = HTTPExecutor( command_exec, 'http://{host}:{port}'.format( host=elasticsearch_host, port=elasticsearch_port ), timeout=60, ) elasticsearch_executor.start() def finalize_elasticsearch(): elasticsearch_executor.stop() shutil.rmtree(work_path) shutil.rmtree(logs_path) request.addfinalizer(finalize_elasticsearch) return elasticsearch_executor
def smoketest(ctx, debug, local_matrix, **kwargs): # pylint: disable=unused-argument """ Test, that the raiden installation is sane. """ import binascii from web3 import Web3, HTTPProvider from web3.middleware import geth_poa_middleware from raiden.api.python import RaidenAPI from raiden.tests.utils.geth import geth_wait_and_check from raiden.tests.integration.contracts.fixtures.contracts import deploy_token from raiden.tests.utils.smoketest import ( TEST_PARTNER_ADDRESS, TEST_DEPOSIT_AMOUNT, deploy_smoketest_contracts, get_private_key, load_smoketest_config, start_ethereum, run_smoketests, ) report_file = tempfile.mktemp(suffix='.log') configure_logging({'': 'DEBUG'}, log_file=report_file) def append_report(subject, data): with open(report_file, 'a', encoding='UTF-8') as handler: handler.write(f'{f" {subject.upper()} ":=^80}{os.linesep}') if data is not None: if isinstance(data, bytes): data = data.decode() handler.writelines([data + os.linesep]) append_report('Raiden version', json.dumps(get_system_spec())) append_report('Raiden log', None) step_count = 7 if ctx.parent.params['transport'] == 'matrix': step_count = 8 step = 0 def print_step(description, error=False): nonlocal step step += 1 click.echo( '{} {}'.format( click.style(f'[{step}/{step_count}]', fg='blue'), click.style(description, fg='green' if not error else 'red'), ), ) print_step('Getting smoketest configuration') smoketest_config = load_smoketest_config() if not smoketest_config: append_report( 'Smoketest configuration', 'Could not load the smoketest genesis configuration file.', ) print_step('Starting Ethereum node') ethereum, ethereum_config = start_ethereum(smoketest_config['genesis']) port = ethereum_config['rpc'] web3_client = Web3(HTTPProvider(f'http://0.0.0.0:{port}')) web3_client.middleware_stack.inject(geth_poa_middleware, layer=0) random_marker = binascii.hexlify(b'raiden').decode() privatekeys = [] geth_wait_and_check(web3_client, privatekeys, random_marker) print_step('Deploying Raiden contracts') host = '0.0.0.0' client = JSONRPCClient( host, ethereum_config['rpc'], get_private_key(), web3=web3_client, ) contract_addresses = deploy_smoketest_contracts(client, 627) token_contract = deploy_token(client) token = token_contract(1000, 0, 'TKN', 'TKN') registry = TokenNetworkRegistry( client, contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY]) registry.add_token(to_canonical_address(token.contract.address)) print_step('Setting up Raiden') # setup cli arguments for starting raiden args = dict( discovery_contract_address=to_checksum_address( contract_addresses[CONTRACT_ENDPOINT_REGISTRY], ), registry_contract_address=to_checksum_address( contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], ), secret_registry_contract_address=to_checksum_address( contract_addresses[CONTRACT_SECRET_REGISTRY], ), eth_rpc_endpoint='http://127.0.0.1:{}'.format(port), keystore_path=ethereum_config['keystore'], address=ethereum_config['address'], network_id='627', transport=ctx.parent.params['transport'], matrix_server='http://localhost:8008' if ctx.parent.params['matrix_server'] == 'auto' else ctx.parent.params['matrix_server'], ) smoketest_config['transport'] = args['transport'] for option_ in app.params: if option_.name in args.keys(): args[option_.name] = option_.process_value(ctx, args[option_.name]) else: args[option_.name] = option_.default password_file = os.path.join(args['keystore_path'], 'password') with open(password_file, 'w') as handler: handler.write('password') port = next(get_free_port('127.0.0.1', 5001)) args['password_file'] = click.File()(password_file) args['datadir'] = args['keystore_path'] args['api_address'] = 'localhost:' + str(port) args['sync_check'] = False def _run_smoketest(): print_step('Starting Raiden') # invoke the raiden app app_ = ctx.invoke(app, **args) raiden_api = RaidenAPI(app_.raiden) rest_api = RestAPI(raiden_api) api_server = APIServer(rest_api) (api_host, api_port) = split_endpoint(args['api_address']) api_server.start(api_host, api_port) raiden_api.channel_open( contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], to_canonical_address(token.contract.address), to_canonical_address(TEST_PARTNER_ADDRESS), None, None, ) raiden_api.set_total_channel_deposit( contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], to_canonical_address(token.contract.address), to_canonical_address(TEST_PARTNER_ADDRESS), TEST_DEPOSIT_AMOUNT, ) smoketest_config['contracts'][ 'registry_address'] = to_checksum_address( contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], ) smoketest_config['contracts'][ 'secret_registry_address'] = to_checksum_address( contract_addresses[CONTRACT_SECRET_REGISTRY], ) smoketest_config['contracts'][ 'discovery_address'] = to_checksum_address( contract_addresses[CONTRACT_ENDPOINT_REGISTRY], ) smoketest_config['contracts']['token_address'] = to_checksum_address( token.contract.address, ) success = False try: print_step('Running smoketest') error = run_smoketests(app_.raiden, smoketest_config, debug=debug) if error is not None: append_report('Smoketest assertion error', error) else: success = True finally: app_.stop() ethereum.send_signal(2) err, out = ethereum.communicate() append_report('Ethereum init stdout', ethereum_config['init_log_out'].decode('utf-8')) append_report('Ethereum init stderr', ethereum_config['init_log_err'].decode('utf-8')) append_report('Ethereum stdout', out) append_report('Ethereum stderr', err) append_report('Smoketest configuration', json.dumps(smoketest_config)) if success: print_step( f'Smoketest successful, report was written to {report_file}') else: print_step( f'Smoketest had errors, report was written to {report_file}', error=True) return success if args['transport'] == 'udp': with SocketFactory('127.0.0.1', port, strategy='none') as mapped_socket: args['mapped_socket'] = mapped_socket success = _run_smoketest() elif args['transport'] == 'matrix' and local_matrix.lower() != 'none': args['mapped_socket'] = None print_step('Starting Matrix transport') try: with HTTPExecutor( local_matrix, status=r'^[24]\d\d$', url=urljoin(args['matrix_server'], '/_matrix/client/versions'), shell=True, ): args['extra_config'] = { 'matrix': { 'discovery_room': { 'server': 'matrix.local.raiden' }, 'server_name': 'matrix.local.raiden', }, } success = _run_smoketest() except (PermissionError, ProcessExitedWithError): append_report('Matrix server start exception', traceback.format_exc()) print_step( f'Error during smoketest setup, report was written to {report_file}', error=True, ) success = False elif args['transport'] == 'matrix' and local_matrix.lower() == "none": args['mapped_socket'] = None success = _run_smoketest() else: # Shouldn't happen raise RuntimeError(f"Invalid transport type '{args['transport']}'") if not success: sys.exit(1)
def smoketest(ctx, debug, local_matrix, **kwargs): # pylint: disable=unused-argument """ Test, that the raiden installation is sane. """ from raiden.api.python import RaidenAPI from raiden.tests.utils.smoketest import ( TEST_PARTNER_ADDRESS, TEST_DEPOSIT_AMOUNT, load_smoketest_config, run_smoketests, setup_testchain_and_raiden, ) report_file = tempfile.mktemp(suffix='.log') configure_logging({'': 'DEBUG'}, log_file=report_file) def append_report(subject, data): with open(report_file, 'a', encoding='UTF-8') as handler: handler.write(f'{f" {subject.upper()} ":=^80}{os.linesep}') if data is not None: if isinstance(data, bytes): data = data.decode() handler.writelines([data + os.linesep]) append_report('Raiden version', json.dumps(get_system_spec())) append_report('Raiden log', None) step_count = 7 if ctx.parent.params['transport'] == 'matrix': step_count = 8 step = 0 def print_step(description, error=False): nonlocal step step += 1 click.echo( '{} {}'.format( click.style(f'[{step}/{step_count}]', fg='blue'), click.style(description, fg='green' if not error else 'red'), ), ) print_step('Getting smoketest configuration') smoketest_config = load_smoketest_config() if not smoketest_config: append_report( 'Smoketest configuration', 'Could not load the smoketest genesis configuration file.', ) result = setup_testchain_and_raiden( smoketest_config, ctx.parent.params['transport'], ctx.parent.params['matrix_server'], print_step, ) args = result['args'] contract_addresses = result['contract_addresses'] token = result['token'] ethereum = result['ethereum'] ethereum_config = result['ethereum_config'] smoketest_config['transport'] = args['transport'] for option_ in run.params: if option_.name in args.keys(): args[option_.name] = option_.process_value(ctx, args[option_.name]) else: args[option_.name] = option_.default port = next(get_free_port('127.0.0.1', 5001)) args['api_address'] = 'localhost:' + str(port) def _run_smoketest(): print_step('Starting Raiden') # invoke the raiden app app = run_app(**args) raiden_api = RaidenAPI(app.raiden) rest_api = RestAPI(raiden_api) api_server = APIServer(rest_api) (api_host, api_port) = split_endpoint(args['api_address']) api_server.start(api_host, api_port) raiden_api.channel_open( contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], to_canonical_address(token.contract.address), to_canonical_address(TEST_PARTNER_ADDRESS), None, None, ) raiden_api.set_total_channel_deposit( contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], to_canonical_address(token.contract.address), to_canonical_address(TEST_PARTNER_ADDRESS), TEST_DEPOSIT_AMOUNT, ) smoketest_config['contracts']['registry_address'] = to_checksum_address( contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], ) smoketest_config['contracts']['secret_registry_address'] = to_checksum_address( contract_addresses[CONTRACT_SECRET_REGISTRY], ) smoketest_config['contracts']['discovery_address'] = to_checksum_address( contract_addresses[CONTRACT_ENDPOINT_REGISTRY], ) smoketest_config['contracts']['token_address'] = to_checksum_address( token.contract.address, ) success = False try: print_step('Running smoketest') error = run_smoketests(app.raiden, smoketest_config, debug=debug) if error is not None: append_report('Smoketest assertion error', error) else: success = True finally: app.stop() ethereum.send_signal(2) err, out = ethereum.communicate() append_report('Ethereum init stdout', ethereum_config['init_log_out'].decode('utf-8')) append_report('Ethereum init stderr', ethereum_config['init_log_err'].decode('utf-8')) append_report('Ethereum stdout', out) append_report('Ethereum stderr', err) append_report('Smoketest configuration', json.dumps(smoketest_config)) if success: print_step(f'Smoketest successful, report was written to {report_file}') else: print_step(f'Smoketest had errors, report was written to {report_file}', error=True) return success if args['transport'] == 'udp': with SocketFactory('127.0.0.1', port, strategy='none') as mapped_socket: args['mapped_socket'] = mapped_socket success = _run_smoketest() elif args['transport'] == 'matrix' and local_matrix.lower() != 'none': args['mapped_socket'] = None print_step('Starting Matrix transport') try: with HTTPExecutor( local_matrix, status=r'^[24]\d\d$', url=urljoin(args['matrix_server'], '/_matrix/client/versions'), shell=True, ): args['extra_config'] = { 'matrix': { 'discovery_room': {'server': 'matrix.local.raiden'}, 'server_name': 'matrix.local.raiden', }, } success = _run_smoketest() except (PermissionError, ProcessExitedWithError): append_report('Matrix server start exception', traceback.format_exc()) print_step( f'Error during smoketest setup, report was written to {report_file}', error=True, ) success = False elif args['transport'] == 'matrix' and local_matrix.lower() == "none": args['mapped_socket'] = None success = _run_smoketest() else: # Shouldn't happen raise RuntimeError(f"Invalid transport type '{args['transport']}'") if not success: sys.exit(1)
def elasticsearch_proc_fixture(request): """Elasticsearch process starting fixture.""" tmpdir = gettempdir() config = return_config(request) version_parts = get_version_parts(executable) elasticsearch_host = host or config['host'] elasticsearch_port = get_port(port) or get_port(config['port']) elasticsearch_transport_port = get_port(transport_tcp_port) or \ get_port(config['transport_tcp_port']) elasticsearch_cluster_name = \ cluster_name or config['cluster_name'] or \ 'elasticsearch_cluster_{0}'.format(elasticsearch_port) elasticsearch_logs_prefix = logs_prefix or config['logs_prefix'] elasticsearch_index_store_type = index_store_type or \ config['index_store_type'] elasticsearch_network_publish_host = network_publish_host or \ config['network_publish_host'] logsdir = elasticsearch_logsdir or config['logsdir'] logs_path = os.path.join( logsdir, '{prefix}elasticsearch_{port}_logs'.format( prefix=elasticsearch_logs_prefix, port=elasticsearch_port)) pidfile = os.path.join( tmpdir, 'elasticsearch.{0}.pid'.format(elasticsearch_port)) home_path = os.path.join( tmpdir, 'elasticsearch_{0}'.format(elasticsearch_port)) work_path = '{0}_tmp'.format(home_path) conf_path = configuration_path or config['configuration_path'] if discovery_zen_ping_multicast_enabled is not None: multicast_enabled = str( discovery_zen_ping_multicast_enabled).lower() else: multicast_enabled = config['discovery_zen_ping_multicast_enabled'] command = command_from(version=(version_parts['major'], version_parts['minor'], version_parts['patch'])) command_exec = command.format( deamon=executable, pidfile=pidfile, port=elasticsearch_port, transport_tcp_port=elasticsearch_transport_port, conf_path=conf_path, home_path=home_path, logs_path=logs_path, work_path=work_path, cluster=elasticsearch_cluster_name, network_publish_host=elasticsearch_network_publish_host, multicast_enabled=multicast_enabled, index_store_type=elasticsearch_index_store_type) elasticsearch_executor = HTTPExecutor( command_exec, 'http://{host}:{port}'.format(host=elasticsearch_host, port=elasticsearch_port), timeout=60, ) elasticsearch_executor.start() def finalize_elasticsearch(): elasticsearch_executor.stop() shutil.rmtree(work_path) shutil.rmtree(logs_path) request.addfinalizer(finalize_elasticsearch) return elasticsearch_executor
def elasticsearch_proc_fixture(request): """Elasticsearch process starting fixture.""" tmpdir = gettempdir() config = return_config(request) es_ver = _get_elastic_version(executable) elasticsearch_host = host or config['host'] elasticsearch_port = get_port(port) or get_port(config['port']) elasticsearch_cluster_name = \ cluster_name or config['cluster_name'] or \ 'elasticsearch_cluster_{0}'.format(elasticsearch_port) elasticsearch_logs_prefix = logs_prefix or config['logs_prefix'] or '' elasticsearch_index_store_type = index_store_type or \ config['index_store_type'] or _default_index_store(es_ver) elasticsearch_network_publish_host = network_publish_host or \ config['network_publish_host'] logsdir = elasticsearch_logsdir or config['logsdir'] or tmpdir logs_path = os.path.join( logsdir, '{prefix}elasticsearch_{port}_logs'.format( prefix=elasticsearch_logs_prefix, port=elasticsearch_port)) data_dir = elasticsearch_datadir or tmpdir conf_path = elasticsearch_confdir or config[ 'confdir'] or _default_conf_dir(es_ver) pidfile = os.path.join( tmpdir, 'elasticsearch.{0}.pid'.format(elasticsearch_port)) home_path = os.path.join( tmpdir, 'elasticsearch_{0}'.format(elasticsearch_port)) work_path = '{0}_tmp'.format(home_path) if discovery_zen_ping_multicast_enabled is not None: multicast_enabled = str( discovery_zen_ping_multicast_enabled).lower() else: multicast_enabled = config['discovery_zen_ping_multicast_enabled'] command_exec = _generate_es_cmdline( es_ver, executable=executable, pid_file=pidfile, host=elasticsearch_host, port=elasticsearch_port, home_path=home_path, logs_path=logs_path, conf_path=conf_path, work_path=work_path, data_path=data_dir, cluster=elasticsearch_cluster_name, network_publish_host=elasticsearch_network_publish_host, multicast_enabled=multicast_enabled, index_store_type=elasticsearch_index_store_type, max_local_storage_nodes=multiprocessing.cpu_count()) elasticsearch_executor = HTTPExecutor(command_exec, 'http://{host}:{port}'.format( host=elasticsearch_host, port=elasticsearch_port), timeout=60) elasticsearch_executor.start() def finalize_elasticsearch(): elasticsearch_executor.stop() try: shutil.rmtree(home_path) except OSError as e: if e.errno != errno.ENOENT: raise request.addfinalizer(finalize_elasticsearch) return elasticsearch_executor
def elasticsearch_proc_fixture(request): """Elasticsearch process starting fixture.""" tmpdir = gettempdir() config = return_config(request) elasticsearch_host = host or config['host'] elasticsearch_port = get_port(port) or get_port(config['port']) elasticsearch_cluster_name = \ cluster_name or config['cluster_name'] or \ 'elasticsearch_cluster_{0}'.format(elasticsearch_port) elasticsearch_logs_prefix = logs_prefix or config['logs_prefix'] elasticsearch_index_store_type = index_store_type or \ config['index_store_type'] elasticsearch_network_publish_host = network_publish_host or \ config['network_publish_host'] logsdir = elasticsearch_logsdir or config['logsdir'] logs_path = os.path.join( logsdir, '{prefix}elasticsearch_{port}_logs'.format( prefix=elasticsearch_logs_prefix, port=elasticsearch_port)) pidfile = os.path.join( tmpdir, 'elasticsearch.{0}.pid'.format(elasticsearch_port)) home_path = os.path.join( tmpdir, 'elasticsearch_{0}'.format(elasticsearch_port)) work_path = '{0}_tmp'.format(home_path) if discovery_zen_ping_multicast_enabled is not None: multicast_enabled = str( discovery_zen_ping_multicast_enabled).lower() else: multicast_enabled = config['discovery_zen_ping_multicast_enabled'] command_exec = ''' {deamon} -p {pidfile} --http.port={port} --path.home={home_path} --default.path.logs={logs_path} --default.path.work={work_path} --default.path.conf=/etc/elasticsearch --cluster.name={cluster} --network.publish_host='{network_publish_host}' --discovery.zen.ping.multicast.enabled={multicast_enabled} --index.store.type={index_store_type} '''.format(deamon=executable, pidfile=pidfile, port=elasticsearch_port, home_path=home_path, logs_path=logs_path, work_path=work_path, cluster=elasticsearch_cluster_name, network_publish_host=elasticsearch_network_publish_host, multicast_enabled=multicast_enabled, index_store_type=elasticsearch_index_store_type) elasticsearch_executor = HTTPExecutor( command_exec, 'http://{host}:{port}'.format(host=elasticsearch_host, port=elasticsearch_port), timeout=60, ) elasticsearch_executor.start() def finalize_elasticsearch(): elasticsearch_executor.stop() shutil.rmtree(home_path) request.addfinalizer(finalize_elasticsearch) return elasticsearch_executor