def local_matrix_server(transport_config):
if not transport_config.protocol == TransportProtocol.MATRIX:
yield None
return
server = transport_config.parameters.server
# if command is none, assume server is already running
if transport_config.parameters.command in (None, 'none'):
yield server
return
# otherwise, run our own local server
matrix = HTTPExecutor(
transport_config.parameters.command,
status=r'^[24]\d\d$',
url=urljoin(server, '/_matrix'),
timeout=120,
sleep=0.1,
shell=True,
)
matrix.start()
yield server
matrix.stop()
def elasticsearch_proc_fixture(request):
"""Elasticsearch process starting fixture."""
elasticsearch_port = get_port(port)
pidfile = '/tmp/elasticsearch.{0}.pid'.format(elasticsearch_port)
home_path = '/tmp/elasticsearch_{0}'.format(elasticsearch_port)
logsdir = path(request.config.getvalue('logsdir'))
logs_path = logsdir / '{prefix}elasticsearch_{port}_logs'.format(
prefix=logs_prefix,
port=elasticsearch_port
)
work_path = '/tmp/elasticsearch_{0}_tmp'.format(elasticsearch_port)
cluster = cluster_name or 'dbfixtures.{0}'.format(elasticsearch_port)
multicast_enabled = str(discovery_zen_ping_multicast_enabled).lower()
command_exec = '''
{deamon} -p {pidfile} --http.port={port}
--path.home={home_path} --default.path.logs={logs_path}
--default.path.work={work_path}
--default.path.conf=/etc/elasticsearch
--cluster.name={cluster}
--network.publish_host='{network_publish_host}'
--discovery.zen.ping.multicast.enabled={multicast_enabled}
--index.store.type={index_store_type}
'''.format(
deamon=executable,
pidfile=pidfile,
port=elasticsearch_port,
home_path=home_path,
logs_path=logs_path,
work_path=work_path,
cluster=cluster,
network_publish_host=network_publish_host,
multicast_enabled=multicast_enabled,
index_store_type=index_store_type
)
elasticsearch_executor = HTTPExecutor(
command_exec, 'http://{host}:{port}'.format(
host=host,
port=elasticsearch_port
),
timeout=60,
)
elasticsearch_executor.start()
def finalize_elasticsearch():
elasticsearch_executor.stop()
shutil.rmtree(home_path)
request.addfinalizer(finalize_elasticsearch)
return elasticsearch_executor
def backend(request):
"""Run application backend."""
executor = HTTPExecutor(
[
'planningpoker',
'--host', '127.0.0.1',
'--port', str(PORT),
'--cookie-secret-key', Fernet.generate_key().decode()
],
SITE_ADDRESS + '/status',
timeout=EXECUTOR_TIMEOUT
)
executor.start()
request.addfinalizer(executor.stop)
def test_shell_started_server_stops():
"""Test if executor terminates properly executor with shell=True."""
executor = HTTPExecutor(http_server_cmd, "http://{0}:{1}/".format(HOST, PORT), timeout=20, shell=True)
with pytest.raises(socket.error):
connect_to_server()
with executor:
assert executor.running() is True
connect_to_server()
assert executor.running() is False
with pytest.raises(socket.error):
connect_to_server()
def test_slow_method_server_timed_out(method):
"""Check if timeout properly expires."""
http_method_slow_cmd = (
f'{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT} False {method}')
executor = HTTPExecutor(http_method_slow_cmd,
f'http://{HOST}:{PORT}/',
method=method,
timeout=1)
with pytest.raises(TimeoutExpired) as exc:
executor.start()
assert executor.running() is False
assert 'timed out after' in str(exc.value)
def test_default_port():
"""
Test default port for the base TCP check.
Check if HTTP executor fills in the default port for the TCP check
from the base class if no port is provided in the URL.
"""
executor = HTTPExecutor(http_server_cmd, 'http://{}/'.format(HOST))
assert executor.url.port is None
assert executor.port == PORT
assert TCPExecutor.pre_start_check(executor) is False
executor.start()
assert TCPExecutor.pre_start_check(executor) is True
executor.stop()
def test_fail_if_other_executor_running():
"""Test raising AlreadyRunning exception when port is blocked."""
executor = HTTPExecutor(http_server_cmd, "http://{0}:{1}/".format(HOST, PORT))
executor2 = HTTPExecutor(http_server_cmd, "http://{0}:{1}/".format(HOST, PORT))
with executor:
assert executor.running() is True
with pytest.raises(AlreadyRunning):
executor2.start()
with pytest.raises(AlreadyRunning) as exc:
with executor2:
pass
assert "seems to be already running" in str(exc)
def test_shell_started_server_stops():
"""Test if executor terminates properly executor with shell=True."""
executor = HTTPExecutor(http_server_cmd,
'http://{0}:{1}/'.format(HOST, PORT),
timeout=20,
shell=True)
with pytest.raises(socket.error):
connect_to_server()
with executor:
assert executor.running() is True
connect_to_server()
assert executor.running() is False
with pytest.raises(socket.error):
connect_to_server()
def test_shell_started_server_stops():
"""Test if executor terminates properly executor with shell=True."""
executor = HTTPExecutor(HTTP_NORMAL_CMD,
f'http://{HOST}:{PORT}/',
timeout=20,
shell=True)
with pytest.raises(socket.error):
connect_to_server()
with executor:
assert executor.running() is True
connect_to_server()
assert executor.running() is False
with pytest.raises(socket.error):
connect_to_server()
def test_fail_if_other_executor_running():
"""Test raising AlreadyRunning exception when port is blocked."""
executor = HTTPExecutor(
http_server_cmd, 'http://{0}:{1}/'.format(HOST, PORT),
)
executor2 = HTTPExecutor(
http_server_cmd, 'http://{0}:{1}/'.format(HOST, PORT),
)
with executor:
assert executor.running() is True
with pytest.raises(AlreadyRunning):
executor2.start()
with pytest.raises(AlreadyRunning):
with executor2:
pass
def local_matrix_server(transport_config):
if not transport_config.protocol == TransportProtocol.MATRIX:
yield None
return
server = transport_config.parameters.server
# if command is none, assume server is already running
if transport_config.parameters.command in (None, 'none'):
yield server
return
# otherwise, run our own local server
matrix = HTTPExecutor(
transport_config.parameters.command,
status=r'^[24]\d\d$',
url=urljoin(server, '/_matrix'),
timeout=120,
sleep=0.1,
shell=True,
)
matrix.start()
yield server
matrix.stop()
def test_http_status_codes(accepted_status, expected_timeout):
"""
Test how 'status' argument influences executor start.
:param int|str accepted_status: Executor 'status' value
:param bool expected_timeout: if Executor raises TimeoutExpired or not
"""
kwargs = {
'command': http_server_cmd,
'url': 'http://{0}:{1}/badpath'.format(HOST, PORT),
'timeout': 2
}
if accepted_status:
kwargs['status'] = accepted_status
executor = HTTPExecutor(**kwargs)
if not expected_timeout:
executor.start()
executor.stop()
else:
with pytest.raises(TimeoutExpired):
executor.start()
executor.stop()
def test_http_status_codes(accepted_status, expected_timeout):
"""
Test how 'status' argument influences executor start.
:param int|str accepted_status: Executor 'status' value
:param bool expected_timeout: if Executor raises TimeoutExpired or not
"""
kwargs: Dict[str, Any] = {
'command': HTTP_NORMAL_CMD,
'url': f'http://{HOST}:{PORT}/badpath',
'timeout': 2
}
if accepted_status:
kwargs['status'] = accepted_status
executor = HTTPExecutor(**kwargs)
if not expected_timeout:
executor.start()
executor.stop()
else:
with pytest.raises(TimeoutExpired):
executor.start()
executor.stop()
def test_stopping_brutally():
"""
Test if SimpleExecutor is stopping insubordinate process.
Check if the process that doesn't react to SIGTERM signal will be killed
by executor with SIGKILL automatically.
"""
host_port = "127.0.0.1:8000"
cmd = f"{sys.executable} {TEST_SERVER_PATH} {host_port} True"
executor = HTTPExecutor(cmd, f"http://{host_port!s}/", timeout=20)
executor.start()
assert executor.running() is True
stop_at = time.time() + 10
executor.stop()
assert executor.running() is False
assert stop_at <= time.time(), "Subprocess killed earlier than in 10 secs"
def test_http_status_codes(accepted_status: Union[None, int, str],
expected_timeout: bool) -> None:
"""
Test how 'status' argument influences executor start.
:param int|str accepted_status: Executor 'status' value
:param bool expected_timeout: if Executor raises TimeoutExpired or not
"""
kwargs: Dict[str, Any] = {
"command": HTTP_NORMAL_CMD,
"url": f"http://{HOST}:{PORT}/badpath",
"timeout": 2,
}
if accepted_status:
kwargs["status"] = accepted_status
executor = HTTPExecutor(**kwargs)
if not expected_timeout:
executor.start()
executor.stop()
else:
with pytest.raises(TimeoutExpired):
executor.start()
executor.stop()
def test_stopping_brutally():
"""
Test if Executor is stopping insubordinate process.
Check if the process that doesn't react to SIGTERM signal will be killed
by executor with SIGKILL automatically.
"""
host_port = "127.0.0.1:8000"
cmd = '{} {} {} True'.format(sys.executable, test_server_path, host_port)
executor = HTTPExecutor(cmd, 'http://%s/' % host_port)
executor.start()
assert executor.running() is True
stop_at = time.time() + 10
executor.stop()
assert executor.running() is False
assert stop_at <= time.time(), "Subprocess killed earlier than in 10 secs"
def test_slow_method_server_starting(method):
"""
Test whether or not executor awaits for slow starting servers.
Simple example. You run Gunicorn and it is working but you have to
wait for worker processes.
"""
http_method_slow_cmd = (
f'{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT} False {method}')
with HTTPExecutor(http_method_slow_cmd,
f'http://{HOST}:{PORT}/',
method=method,
timeout=30) as executor:
assert executor.running() is True
connect_to_server()
def test_executor_starts_and_waits():
"""Test if process awaits for HEAD request to be completed."""
command = f'bash -c "sleep 3 && {HTTP_NORMAL_CMD}"'
executor = HTTPExecutor(command, f'http://{HOST}:{PORT}/', timeout=20)
executor.start()
assert executor.running() is True
connect_to_server()
executor.stop()
# check proper __str__ and __repr__ rendering:
assert 'HTTPExecutor' in repr(executor)
assert command in str(executor)
def test_executor_starts_and_waits():
"""Test if process awaits for HEAD request to be completed."""
command = 'bash -c "sleep 3 && {0}"'.format(http_server_cmd)
executor = HTTPExecutor(command,
'http://{0}:{1}/'.format(HOST, PORT),
timeout=20)
executor.start()
assert executor.running() is True
connect_to_server()
executor.stop()
# check proper __str__ and __repr__ rendering:
assert 'HTTPExecutor' in repr(executor)
assert command in str(executor)
def test_stopping_brutally():
"""
Test if SimpleExecutor is stopping insubordinate process.
Check if the process that doesn't react to SIGTERM signal will be killed
by executor with SIGKILL automatically.
"""
host_port = "127.0.0.1:8000"
cmd = '{} {} {} True'.format(sys.executable, test_server_path, host_port)
executor = HTTPExecutor(cmd, 'http://%s/' % host_port)
executor.start()
assert executor.running() is True
stop_at = time.time() + 10
executor.stop()
assert executor.running() is False
assert stop_at <= time.time(), "Subprocess killed earlier than in 10 secs"
def test_slow_post_payload_server_starting():
"""
Test whether or not executor awaits for slow starting servers.
Simple example. You run Gunicorn and it is working but you have to
wait for worker processes.
"""
http_method_slow_cmd = (
f"{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT} False Key")
with HTTPExecutor(
http_method_slow_cmd,
f"http://{HOST}:{PORT}/",
method="POST",
timeout=30,
payload={"key": "hole"},
) as executor:
assert executor.running() is True
connect_to_server()
def test_executor_starts_and_waits():
"""Test if process awaits for HEAD request to be completed."""
command = 'bash -c "sleep 3 && {0}"'.format(http_server_cmd)
executor = HTTPExecutor(
command, 'http://{0}:{1}/'.format(HOST, PORT),
timeout=20
)
executor.start()
assert executor.running() is True
connect_to_server()
executor.stop()
def test_executor_starts_and_waits():
"""Test if process awaits for HEAD request to be completed."""
command = 'bash -c "sleep 3 && {0}"'.format(http_server_cmd)
executor = HTTPExecutor(command, "http://{0}:{1}/".format(HOST, PORT), timeout=20)
executor.start()
assert executor.running() is True
connect_to_server()
executor.stop()
# check proper __str__ and __repr__ rendering:
assert "HTTPExecutor" in repr(executor)
assert command in str(executor)
def test_default_port():
"""
Test default port for the base TCP check.
Check if HTTP executor fills in the default port for the TCP check
from the base class if no port is provided in the URL.
"""
executor = HTTPExecutor(http_server_cmd, 'http://{0}/'.format(HOST))
assert executor.url.port is None
assert executor.port == PORT
assert TCPExecutor.pre_start_check(executor) is False
executor.start()
assert TCPExecutor.pre_start_check(executor) is True
executor.stop()
def test_default_port():
"""
Test default port for the base TCP check.
Check if HTTP executor fills in the default port for the TCP check
from the base class if no port is provided in the URL.
"""
executor = HTTPExecutor(HTTP_NORMAL_CMD, f'http://{HOST}/')
assert executor.url.port is None
assert executor.port == PORT
assert TCPExecutor.pre_start_check(executor) is False
executor.start()
assert TCPExecutor.pre_start_check(executor) is True
executor.stop()
def es_process(es_binary, es_version):
port = port_for.select_random()
pid = es_binary + '.pid'
# use a different garbage collector for better performance
os.environ['ES_JAVA_OPTS'] = \
'-Xms1g -Xmx1g -XX:-UseConcMarkSweepGC -XX:+UseG1GC'
command = (
f"{es_binary} -p {pid} -E http.port={port} "
f"-E xpack.monitoring.enabled=false "
f"-E xpack.monitoring.collection.enabled=false "
f"> /dev/null"
)
url = f'http://127.0.0.1:{port}/_cluster/health?wait_for_status=green'
executor = HTTPExecutor(command, url, method='GET', shell=True)
executor.start()
yield executor
executor.stop()
executor.kill()
def test_fail_if_other_executor_running():
"""Test raising AlreadyRunning exception when port is blocked."""
executor = HTTPExecutor(
http_server_cmd,
'http://{0}:{1}/'.format(HOST, PORT),
)
executor2 = HTTPExecutor(
http_server_cmd,
'http://{0}:{1}/'.format(HOST, PORT),
)
with executor:
assert executor.running() is True
with pytest.raises(AlreadyRunning):
executor2.start()
with pytest.raises(AlreadyRunning) as exc:
with executor2:
pass
assert 'seems to be already running' in str(exc)
def test_fail_if_other_running():
"""Test raising AlreadyRunning exception when port is blocked."""
executor = HTTPExecutor(
HTTP_NORMAL_CMD,
f'http://{HOST}:{PORT}/',
)
executor2 = HTTPExecutor(
HTTP_NORMAL_CMD,
f'http://{HOST}:{PORT}/',
)
with executor:
assert executor.running() is True
with pytest.raises(AlreadyRunning):
executor2.start()
with pytest.raises(AlreadyRunning) as exc:
with executor2:
pass
assert 'seems to be already running' in str(exc.value)
def elasticsearch_proc_fixture(request):
"""Elasticsearch process starting fixture."""
tmpdir = gettempdir()
config = return_config(request)
elasticsearch_host = host or config["host"]
elasticsearch_port = get_port(port) or get_port(config["port"])
elasticsearch_cluster_name = (
cluster_name or config["cluster_name"] or "elasticsearch_cluster_{0}".format(elasticsearch_port)
)
elasticsearch_logs_prefix = logs_prefix or config["logs_prefix"]
elasticsearch_index_store_type = index_store_type or config["index_store_type"]
elasticsearch_network_publish_host = network_publish_host or config["network_publish_host"]
logsdir = elasticsearch_logsdir or config["logsdir"]
logs_path = os.path.join(
logsdir,
"{prefix}elasticsearch_{port}_logs".format(prefix=elasticsearch_logs_prefix, port=elasticsearch_port),
)
pidfile = os.path.join(tmpdir, "elasticsearch.{0}.pid".format(elasticsearch_port))
home_path = os.path.join(tmpdir, "elasticsearch_{0}".format(elasticsearch_port))
work_path = "{0}_tmp".format(home_path)
if discovery_zen_ping_multicast_enabled is not None:
multicast_enabled = str(discovery_zen_ping_multicast_enabled).lower()
else:
multicast_enabled = config["discovery_zen_ping_multicast_enabled"]
command_exec = """
{deamon} -p {pidfile} --http.port={port}
--path.home={home_path} --default.path.logs={logs_path}
--default.path.work={work_path}
--default.path.conf=/etc/elasticsearch
--cluster.name={cluster}
--network.publish_host='{network_publish_host}'
--discovery.zen.ping.multicast.enabled={multicast_enabled}
--index.store.type={index_store_type}
""".format(
deamon=executable,
pidfile=pidfile,
port=elasticsearch_port,
home_path=home_path,
logs_path=logs_path,
work_path=work_path,
cluster=elasticsearch_cluster_name,
network_publish_host=elasticsearch_network_publish_host,
multicast_enabled=multicast_enabled,
index_store_type=elasticsearch_index_store_type,
)
elasticsearch_executor = HTTPExecutor(
command_exec, "http://{host}:{port}".format(host=elasticsearch_host, port=elasticsearch_port), timeout=60
)
elasticsearch_executor.start()
def finalize_elasticsearch():
elasticsearch_executor.stop()
shutil.rmtree(home_path)
request.addfinalizer(finalize_elasticsearch)
return elasticsearch_executor
def smoketest(ctx, debug, local_matrix, **kwargs): # pylint: disable=unused-argument
""" Test, that the raiden installation is sane. """
import binascii
from raiden.api.python import RaidenAPI
from raiden.blockchain.abi import get_static_or_compile
from raiden.tests.utils.blockchain import geth_wait_and_check
from raiden.tests.integration.fixtures.backend_geth import web3
from raiden.tests.utils.smoketest import (
load_smoketest_config,
start_ethereum,
run_smoketests,
patch_smoke_fns,
)
from raiden.utils import get_contract_path
from raiden.raiden_service import RaidenService
# TODO: Temporary until we also deploy contracts in smoketests
# This function call patches the initial query filtering to create some fake
# events. That is done to make up for the missing events that would have
# been generated and populated the state if the contracts were deployed
# and not precompiled in the smoketest genesis file
RaidenService.install_and_query_payment_network_filters = patch_smoke_fns(
RaidenService.install_and_query_payment_network_filters, )
# Check the solidity compiler early in the smoketest.
#
# Binary distributions don't need the solidity compiler but source
# distributions do. Since this is checked by `get_static_or_compile`
# function, use it as a proxy for validating the setup.
get_static_or_compile(
get_contract_path('HumanStandardToken.sol'),
'HumanStandardToken',
)
report_file = tempfile.mktemp(suffix='.log')
configure_logging({'': 'DEBUG'}, log_file=report_file)
def append_report(subject, data):
with open(report_file, 'a', encoding='UTF-8') as handler:
handler.write(f'{f" {subject.upper()} ":=^80}{os.linesep}')
if data is not None:
if isinstance(data, bytes):
data = data.decode()
handler.writelines([data + os.linesep])
append_report('raiden version', json.dumps(get_system_spec()))
append_report('raiden log', None)
print('[1/5] getting smoketest configuration')
smoketest_config = load_smoketest_config()
if not smoketest_config:
append_report(
'smoketest configuration',
'Could not load the smoketest genesis configuration file.',
)
print('[2/5] starting ethereum')
ethereum, ethereum_config = start_ethereum(smoketest_config['genesis'])
port = ethereum_config['rpc']
web3_client = web3([port])
random_marker = binascii.hexlify(b'raiden').decode()
privatekeys = []
geth_wait_and_check(web3_client, privatekeys, random_marker)
print('[3/5] starting raiden')
# setup cli arguments for starting raiden
args = dict(
discovery_contract_address=smoketest_config['contracts']
['discovery_address'],
registry_contract_address=smoketest_config['contracts']
['registry_address'],
secret_registry_contract_address=smoketest_config['contracts']
['secret_registry_address'],
eth_rpc_endpoint='http://127.0.0.1:{}'.format(port),
keystore_path=ethereum_config['keystore'],
address=ethereum_config['address'],
network_id='627',
transport=ctx.parent.params['transport'],
matrix_server='http://localhost:8008'
if ctx.parent.params['matrix_server'] == 'auto' else
ctx.parent.params['matrix_server'],
)
smoketest_config['transport'] = args['transport']
for option_ in app.params:
if option_.name in args.keys():
args[option_.name] = option_.process_value(ctx, args[option_.name])
else:
args[option_.name] = option_.default
password_file = os.path.join(args['keystore_path'], 'password')
with open(password_file, 'w') as handler:
handler.write('password')
port = next(get_free_port('127.0.0.1', 5001))
args['password_file'] = click.File()(password_file)
args['datadir'] = args['keystore_path']
args['api_address'] = 'localhost:' + str(port)
args['sync_check'] = False
def _run_smoketest():
# invoke the raiden app
app_ = ctx.invoke(app, **args)
raiden_api = RaidenAPI(app_.raiden)
rest_api = RestAPI(raiden_api)
api_server = APIServer(rest_api)
(api_host, api_port) = split_endpoint(args['api_address'])
api_server.start(api_host, api_port)
success = False
try:
print('[4/5] running smoketests...')
error = run_smoketests(app_.raiden, smoketest_config, debug=debug)
if error is not None:
append_report('smoketest assertion error', error)
else:
success = True
finally:
app_.stop()
ethereum.send_signal(2)
err, out = ethereum.communicate()
append_report('geth init stdout',
ethereum_config['init_log_out'].decode('utf-8'))
append_report('geth init stderr',
ethereum_config['init_log_err'].decode('utf-8'))
append_report('ethereum stdout', out)
append_report('ethereum stderr', err)
append_report('smoketest configuration',
json.dumps(smoketest_config))
if success:
print(
'[5/5] smoketest successful, report was written to {}'.format(
report_file))
else:
print(
'[5/5] smoketest had errors, report was written to {}'.format(
report_file))
return success
if args['transport'] == 'udp':
with SocketFactory('127.0.0.1', port,
strategy='none') as mapped_socket:
args['mapped_socket'] = mapped_socket
success = _run_smoketest()
elif args['transport'] == 'matrix' and local_matrix.lower() != 'none':
print('WARNING: The Matrix transport is experimental')
args['mapped_socket'] = None
with HTTPExecutor(
local_matrix,
status=r'^[24]\d\d$',
url=urljoin(args['matrix_server'], '/_matrix'),
):
args['extra_config'] = {
'matrix': {
'discovery_room': {
'server': 'matrix.local.raiden'
},
'server_name': 'matrix.local.raiden',
},
}
success = _run_smoketest()
elif args['transport'] == 'matrix' and local_matrix.lower() == "none":
print('WARNING: The Matrix transport is experimental')
args['mapped_socket'] = None
success = _run_smoketest()
else:
# Shouldn't happen
raise RuntimeError(f"Invalid transport type '{args['transport']}'")
if not success:
sys.exit(1)
def elasticsearch_proc_fixture(request):
"""Elasticsearch process starting fixture."""
config = return_config(request)
elasticsearch_host = host or config['host']
elasticsearch_port = get_port(port) or get_port(config['port'])
elasticsearch_transport_port = get_port(transport_tcp_port) or \
get_port(config['transport_tcp_port'])
elasticsearch_cluster_name = \
cluster_name or config['cluster_name'] or \
'elasticsearch_cluster_{0}'.format(elasticsearch_port)
elasticsearch_logs_prefix = logs_prefix or config['logs_prefix']
elasticsearch_index_store_type = index_store_type or \
config['index_store_type']
elasticsearch_network_publish_host = network_publish_host or \
config['network_publish_host']
logsdir = elasticsearch_logsdir or config['logsdir']
logs_path = os.path.join(
logsdir, '{prefix}elasticsearch_{port}_logs'.format(
prefix=elasticsearch_logs_prefix,
port=elasticsearch_port
))
pidfile = os.path.join(
gettempdir(), 'elasticsearch.{0}.pid'.format(elasticsearch_port))
home_path = os.path.join(
gettempdir(), 'elasticsearch_{0}'.format(elasticsearch_port))
work_path = '{0}_tmp'.format(home_path)
conf_path = configuration_path or config['configuration_path']
if discovery_zen_ping_multicast_enabled is not None:
multicast_enabled = str(
discovery_zen_ping_multicast_enabled).lower()
else:
multicast_enabled = config['discovery_zen_ping_multicast_enabled']
command = command_from(version=get_version_parts(executable))
command_exec = command.format(
deamon=executable,
pidfile=pidfile,
port=elasticsearch_port,
transport_tcp_port=elasticsearch_transport_port,
conf_path=conf_path,
home_path=home_path,
logs_path=logs_path,
work_path=work_path,
cluster=elasticsearch_cluster_name,
network_publish_host=elasticsearch_network_publish_host,
multicast_enabled=multicast_enabled,
index_store_type=elasticsearch_index_store_type
)
elasticsearch_executor = HTTPExecutor(
command_exec, 'http://{host}:{port}'.format(
host=elasticsearch_host,
port=elasticsearch_port
),
timeout=60,
)
elasticsearch_executor.start()
def finalize_elasticsearch():
elasticsearch_executor.stop()
shutil.rmtree(work_path)
shutil.rmtree(logs_path)
request.addfinalizer(finalize_elasticsearch)
return elasticsearch_executor
def smoketest(ctx, debug, local_matrix, **kwargs): # pylint: disable=unused-argument
""" Test, that the raiden installation is sane. """
import binascii
from web3 import Web3, HTTPProvider
from web3.middleware import geth_poa_middleware
from raiden.api.python import RaidenAPI
from raiden.tests.utils.geth import geth_wait_and_check
from raiden.tests.integration.contracts.fixtures.contracts import deploy_token
from raiden.tests.utils.smoketest import (
TEST_PARTNER_ADDRESS,
TEST_DEPOSIT_AMOUNT,
deploy_smoketest_contracts,
get_private_key,
load_smoketest_config,
start_ethereum,
run_smoketests,
)
report_file = tempfile.mktemp(suffix='.log')
configure_logging({'': 'DEBUG'}, log_file=report_file)
def append_report(subject, data):
with open(report_file, 'a', encoding='UTF-8') as handler:
handler.write(f'{f" {subject.upper()} ":=^80}{os.linesep}')
if data is not None:
if isinstance(data, bytes):
data = data.decode()
handler.writelines([data + os.linesep])
append_report('Raiden version', json.dumps(get_system_spec()))
append_report('Raiden log', None)
step_count = 7
if ctx.parent.params['transport'] == 'matrix':
step_count = 8
step = 0
def print_step(description, error=False):
nonlocal step
step += 1
click.echo(
'{} {}'.format(
click.style(f'[{step}/{step_count}]', fg='blue'),
click.style(description, fg='green' if not error else 'red'),
), )
print_step('Getting smoketest configuration')
smoketest_config = load_smoketest_config()
if not smoketest_config:
append_report(
'Smoketest configuration',
'Could not load the smoketest genesis configuration file.',
)
print_step('Starting Ethereum node')
ethereum, ethereum_config = start_ethereum(smoketest_config['genesis'])
port = ethereum_config['rpc']
web3_client = Web3(HTTPProvider(f'http://0.0.0.0:{port}'))
web3_client.middleware_stack.inject(geth_poa_middleware, layer=0)
random_marker = binascii.hexlify(b'raiden').decode()
privatekeys = []
geth_wait_and_check(web3_client, privatekeys, random_marker)
print_step('Deploying Raiden contracts')
host = '0.0.0.0'
client = JSONRPCClient(
host,
ethereum_config['rpc'],
get_private_key(),
web3=web3_client,
)
contract_addresses = deploy_smoketest_contracts(client, 627)
token_contract = deploy_token(client)
token = token_contract(1000, 0, 'TKN', 'TKN')
registry = TokenNetworkRegistry(
client, contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY])
registry.add_token(to_canonical_address(token.contract.address))
print_step('Setting up Raiden')
# setup cli arguments for starting raiden
args = dict(
discovery_contract_address=to_checksum_address(
contract_addresses[CONTRACT_ENDPOINT_REGISTRY], ),
registry_contract_address=to_checksum_address(
contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], ),
secret_registry_contract_address=to_checksum_address(
contract_addresses[CONTRACT_SECRET_REGISTRY], ),
eth_rpc_endpoint='http://127.0.0.1:{}'.format(port),
keystore_path=ethereum_config['keystore'],
address=ethereum_config['address'],
network_id='627',
transport=ctx.parent.params['transport'],
matrix_server='http://localhost:8008'
if ctx.parent.params['matrix_server'] == 'auto' else
ctx.parent.params['matrix_server'],
)
smoketest_config['transport'] = args['transport']
for option_ in app.params:
if option_.name in args.keys():
args[option_.name] = option_.process_value(ctx, args[option_.name])
else:
args[option_.name] = option_.default
password_file = os.path.join(args['keystore_path'], 'password')
with open(password_file, 'w') as handler:
handler.write('password')
port = next(get_free_port('127.0.0.1', 5001))
args['password_file'] = click.File()(password_file)
args['datadir'] = args['keystore_path']
args['api_address'] = 'localhost:' + str(port)
args['sync_check'] = False
def _run_smoketest():
print_step('Starting Raiden')
# invoke the raiden app
app_ = ctx.invoke(app, **args)
raiden_api = RaidenAPI(app_.raiden)
rest_api = RestAPI(raiden_api)
api_server = APIServer(rest_api)
(api_host, api_port) = split_endpoint(args['api_address'])
api_server.start(api_host, api_port)
raiden_api.channel_open(
contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY],
to_canonical_address(token.contract.address),
to_canonical_address(TEST_PARTNER_ADDRESS),
None,
None,
)
raiden_api.set_total_channel_deposit(
contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY],
to_canonical_address(token.contract.address),
to_canonical_address(TEST_PARTNER_ADDRESS),
TEST_DEPOSIT_AMOUNT,
)
smoketest_config['contracts'][
'registry_address'] = to_checksum_address(
contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], )
smoketest_config['contracts'][
'secret_registry_address'] = to_checksum_address(
contract_addresses[CONTRACT_SECRET_REGISTRY], )
smoketest_config['contracts'][
'discovery_address'] = to_checksum_address(
contract_addresses[CONTRACT_ENDPOINT_REGISTRY], )
smoketest_config['contracts']['token_address'] = to_checksum_address(
token.contract.address, )
success = False
try:
print_step('Running smoketest')
error = run_smoketests(app_.raiden, smoketest_config, debug=debug)
if error is not None:
append_report('Smoketest assertion error', error)
else:
success = True
finally:
app_.stop()
ethereum.send_signal(2)
err, out = ethereum.communicate()
append_report('Ethereum init stdout',
ethereum_config['init_log_out'].decode('utf-8'))
append_report('Ethereum init stderr',
ethereum_config['init_log_err'].decode('utf-8'))
append_report('Ethereum stdout', out)
append_report('Ethereum stderr', err)
append_report('Smoketest configuration',
json.dumps(smoketest_config))
if success:
print_step(
f'Smoketest successful, report was written to {report_file}')
else:
print_step(
f'Smoketest had errors, report was written to {report_file}',
error=True)
return success
if args['transport'] == 'udp':
with SocketFactory('127.0.0.1', port,
strategy='none') as mapped_socket:
args['mapped_socket'] = mapped_socket
success = _run_smoketest()
elif args['transport'] == 'matrix' and local_matrix.lower() != 'none':
args['mapped_socket'] = None
print_step('Starting Matrix transport')
try:
with HTTPExecutor(
local_matrix,
status=r'^[24]\d\d$',
url=urljoin(args['matrix_server'],
'/_matrix/client/versions'),
shell=True,
):
args['extra_config'] = {
'matrix': {
'discovery_room': {
'server': 'matrix.local.raiden'
},
'server_name': 'matrix.local.raiden',
},
}
success = _run_smoketest()
except (PermissionError, ProcessExitedWithError):
append_report('Matrix server start exception',
traceback.format_exc())
print_step(
f'Error during smoketest setup, report was written to {report_file}',
error=True,
)
success = False
elif args['transport'] == 'matrix' and local_matrix.lower() == "none":
args['mapped_socket'] = None
success = _run_smoketest()
else:
# Shouldn't happen
raise RuntimeError(f"Invalid transport type '{args['transport']}'")
if not success:
sys.exit(1)
def smoketest(ctx, debug, local_matrix, **kwargs): # pylint: disable=unused-argument
""" Test, that the raiden installation is sane. """
from raiden.api.python import RaidenAPI
from raiden.tests.utils.smoketest import (
TEST_PARTNER_ADDRESS,
TEST_DEPOSIT_AMOUNT,
load_smoketest_config,
run_smoketests,
setup_testchain_and_raiden,
)
report_file = tempfile.mktemp(suffix='.log')
configure_logging({'': 'DEBUG'}, log_file=report_file)
def append_report(subject, data):
with open(report_file, 'a', encoding='UTF-8') as handler:
handler.write(f'{f" {subject.upper()} ":=^80}{os.linesep}')
if data is not None:
if isinstance(data, bytes):
data = data.decode()
handler.writelines([data + os.linesep])
append_report('Raiden version', json.dumps(get_system_spec()))
append_report('Raiden log', None)
step_count = 7
if ctx.parent.params['transport'] == 'matrix':
step_count = 8
step = 0
def print_step(description, error=False):
nonlocal step
step += 1
click.echo(
'{} {}'.format(
click.style(f'[{step}/{step_count}]', fg='blue'),
click.style(description, fg='green' if not error else 'red'),
),
)
print_step('Getting smoketest configuration')
smoketest_config = load_smoketest_config()
if not smoketest_config:
append_report(
'Smoketest configuration',
'Could not load the smoketest genesis configuration file.',
)
result = setup_testchain_and_raiden(
smoketest_config,
ctx.parent.params['transport'],
ctx.parent.params['matrix_server'],
print_step,
)
args = result['args']
contract_addresses = result['contract_addresses']
token = result['token']
ethereum = result['ethereum']
ethereum_config = result['ethereum_config']
smoketest_config['transport'] = args['transport']
for option_ in run.params:
if option_.name in args.keys():
args[option_.name] = option_.process_value(ctx, args[option_.name])
else:
args[option_.name] = option_.default
port = next(get_free_port('127.0.0.1', 5001))
args['api_address'] = 'localhost:' + str(port)
def _run_smoketest():
print_step('Starting Raiden')
# invoke the raiden app
app = run_app(**args)
raiden_api = RaidenAPI(app.raiden)
rest_api = RestAPI(raiden_api)
api_server = APIServer(rest_api)
(api_host, api_port) = split_endpoint(args['api_address'])
api_server.start(api_host, api_port)
raiden_api.channel_open(
contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY],
to_canonical_address(token.contract.address),
to_canonical_address(TEST_PARTNER_ADDRESS),
None,
None,
)
raiden_api.set_total_channel_deposit(
contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY],
to_canonical_address(token.contract.address),
to_canonical_address(TEST_PARTNER_ADDRESS),
TEST_DEPOSIT_AMOUNT,
)
smoketest_config['contracts']['registry_address'] = to_checksum_address(
contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY],
)
smoketest_config['contracts']['secret_registry_address'] = to_checksum_address(
contract_addresses[CONTRACT_SECRET_REGISTRY],
)
smoketest_config['contracts']['discovery_address'] = to_checksum_address(
contract_addresses[CONTRACT_ENDPOINT_REGISTRY],
)
smoketest_config['contracts']['token_address'] = to_checksum_address(
token.contract.address,
)
success = False
try:
print_step('Running smoketest')
error = run_smoketests(app.raiden, smoketest_config, debug=debug)
if error is not None:
append_report('Smoketest assertion error', error)
else:
success = True
finally:
app.stop()
ethereum.send_signal(2)
err, out = ethereum.communicate()
append_report('Ethereum init stdout', ethereum_config['init_log_out'].decode('utf-8'))
append_report('Ethereum init stderr', ethereum_config['init_log_err'].decode('utf-8'))
append_report('Ethereum stdout', out)
append_report('Ethereum stderr', err)
append_report('Smoketest configuration', json.dumps(smoketest_config))
if success:
print_step(f'Smoketest successful, report was written to {report_file}')
else:
print_step(f'Smoketest had errors, report was written to {report_file}', error=True)
return success
if args['transport'] == 'udp':
with SocketFactory('127.0.0.1', port, strategy='none') as mapped_socket:
args['mapped_socket'] = mapped_socket
success = _run_smoketest()
elif args['transport'] == 'matrix' and local_matrix.lower() != 'none':
args['mapped_socket'] = None
print_step('Starting Matrix transport')
try:
with HTTPExecutor(
local_matrix,
status=r'^[24]\d\d$',
url=urljoin(args['matrix_server'], '/_matrix/client/versions'),
shell=True,
):
args['extra_config'] = {
'matrix': {
'discovery_room': {'server': 'matrix.local.raiden'},
'server_name': 'matrix.local.raiden',
},
}
success = _run_smoketest()
except (PermissionError, ProcessExitedWithError):
append_report('Matrix server start exception', traceback.format_exc())
print_step(
f'Error during smoketest setup, report was written to {report_file}',
error=True,
)
success = False
elif args['transport'] == 'matrix' and local_matrix.lower() == "none":
args['mapped_socket'] = None
success = _run_smoketest()
else:
# Shouldn't happen
raise RuntimeError(f"Invalid transport type '{args['transport']}'")
if not success:
sys.exit(1)
def elasticsearch_proc_fixture(request):
"""Elasticsearch process starting fixture."""
tmpdir = gettempdir()
config = return_config(request)
version_parts = get_version_parts(executable)
elasticsearch_host = host or config['host']
elasticsearch_port = get_port(port) or get_port(config['port'])
elasticsearch_transport_port = get_port(transport_tcp_port) or \
get_port(config['transport_tcp_port'])
elasticsearch_cluster_name = \
cluster_name or config['cluster_name'] or \
'elasticsearch_cluster_{0}'.format(elasticsearch_port)
elasticsearch_logs_prefix = logs_prefix or config['logs_prefix']
elasticsearch_index_store_type = index_store_type or \
config['index_store_type']
elasticsearch_network_publish_host = network_publish_host or \
config['network_publish_host']
logsdir = elasticsearch_logsdir or config['logsdir']
logs_path = os.path.join(
logsdir, '{prefix}elasticsearch_{port}_logs'.format(
prefix=elasticsearch_logs_prefix, port=elasticsearch_port))
pidfile = os.path.join(
tmpdir, 'elasticsearch.{0}.pid'.format(elasticsearch_port))
home_path = os.path.join(
tmpdir, 'elasticsearch_{0}'.format(elasticsearch_port))
work_path = '{0}_tmp'.format(home_path)
conf_path = configuration_path or config['configuration_path']
if discovery_zen_ping_multicast_enabled is not None:
multicast_enabled = str(
discovery_zen_ping_multicast_enabled).lower()
else:
multicast_enabled = config['discovery_zen_ping_multicast_enabled']
command = command_from(version=(version_parts['major'],
version_parts['minor'],
version_parts['patch']))
command_exec = command.format(
deamon=executable,
pidfile=pidfile,
port=elasticsearch_port,
transport_tcp_port=elasticsearch_transport_port,
conf_path=conf_path,
home_path=home_path,
logs_path=logs_path,
work_path=work_path,
cluster=elasticsearch_cluster_name,
network_publish_host=elasticsearch_network_publish_host,
multicast_enabled=multicast_enabled,
index_store_type=elasticsearch_index_store_type)
elasticsearch_executor = HTTPExecutor(
command_exec,
'http://{host}:{port}'.format(host=elasticsearch_host,
port=elasticsearch_port),
timeout=60,
)
elasticsearch_executor.start()
def finalize_elasticsearch():
elasticsearch_executor.stop()
shutil.rmtree(work_path)
shutil.rmtree(logs_path)
request.addfinalizer(finalize_elasticsearch)
return elasticsearch_executor
def elasticsearch_proc_fixture(request):
"""Elasticsearch process starting fixture."""
tmpdir = gettempdir()
config = return_config(request)
es_ver = _get_elastic_version(executable)
elasticsearch_host = host or config['host']
elasticsearch_port = get_port(port) or get_port(config['port'])
elasticsearch_cluster_name = \
cluster_name or config['cluster_name'] or \
'elasticsearch_cluster_{0}'.format(elasticsearch_port)
elasticsearch_logs_prefix = logs_prefix or config['logs_prefix'] or ''
elasticsearch_index_store_type = index_store_type or \
config['index_store_type'] or _default_index_store(es_ver)
elasticsearch_network_publish_host = network_publish_host or \
config['network_publish_host']
logsdir = elasticsearch_logsdir or config['logsdir'] or tmpdir
logs_path = os.path.join(
logsdir, '{prefix}elasticsearch_{port}_logs'.format(
prefix=elasticsearch_logs_prefix, port=elasticsearch_port))
data_dir = elasticsearch_datadir or tmpdir
conf_path = elasticsearch_confdir or config[
'confdir'] or _default_conf_dir(es_ver)
pidfile = os.path.join(
tmpdir, 'elasticsearch.{0}.pid'.format(elasticsearch_port))
home_path = os.path.join(
tmpdir, 'elasticsearch_{0}'.format(elasticsearch_port))
work_path = '{0}_tmp'.format(home_path)
if discovery_zen_ping_multicast_enabled is not None:
multicast_enabled = str(
discovery_zen_ping_multicast_enabled).lower()
else:
multicast_enabled = config['discovery_zen_ping_multicast_enabled']
command_exec = _generate_es_cmdline(
es_ver,
executable=executable,
pid_file=pidfile,
host=elasticsearch_host,
port=elasticsearch_port,
home_path=home_path,
logs_path=logs_path,
conf_path=conf_path,
work_path=work_path,
data_path=data_dir,
cluster=elasticsearch_cluster_name,
network_publish_host=elasticsearch_network_publish_host,
multicast_enabled=multicast_enabled,
index_store_type=elasticsearch_index_store_type,
max_local_storage_nodes=multiprocessing.cpu_count())
elasticsearch_executor = HTTPExecutor(command_exec,
'http://{host}:{port}'.format(
host=elasticsearch_host,
port=elasticsearch_port),
timeout=60)
elasticsearch_executor.start()
def finalize_elasticsearch():
elasticsearch_executor.stop()
try:
shutil.rmtree(home_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
request.addfinalizer(finalize_elasticsearch)
return elasticsearch_executor
def elasticsearch_proc_fixture(request):
"""Elasticsearch process starting fixture."""
tmpdir = gettempdir()
config = return_config(request)
elasticsearch_host = host or config['host']
elasticsearch_port = get_port(port) or get_port(config['port'])
elasticsearch_cluster_name = \
cluster_name or config['cluster_name'] or \
'elasticsearch_cluster_{0}'.format(elasticsearch_port)
elasticsearch_logs_prefix = logs_prefix or config['logs_prefix']
elasticsearch_index_store_type = index_store_type or \
config['index_store_type']
elasticsearch_network_publish_host = network_publish_host or \
config['network_publish_host']
logsdir = elasticsearch_logsdir or config['logsdir']
logs_path = os.path.join(
logsdir, '{prefix}elasticsearch_{port}_logs'.format(
prefix=elasticsearch_logs_prefix, port=elasticsearch_port))
pidfile = os.path.join(
tmpdir, 'elasticsearch.{0}.pid'.format(elasticsearch_port))
home_path = os.path.join(
tmpdir, 'elasticsearch_{0}'.format(elasticsearch_port))
work_path = '{0}_tmp'.format(home_path)
if discovery_zen_ping_multicast_enabled is not None:
multicast_enabled = str(
discovery_zen_ping_multicast_enabled).lower()
else:
multicast_enabled = config['discovery_zen_ping_multicast_enabled']
command_exec = '''
{deamon} -p {pidfile} --http.port={port}
--path.home={home_path} --default.path.logs={logs_path}
--default.path.work={work_path}
--default.path.conf=/etc/elasticsearch
--cluster.name={cluster}
--network.publish_host='{network_publish_host}'
--discovery.zen.ping.multicast.enabled={multicast_enabled}
--index.store.type={index_store_type}
'''.format(deamon=executable,
pidfile=pidfile,
port=elasticsearch_port,
home_path=home_path,
logs_path=logs_path,
work_path=work_path,
cluster=elasticsearch_cluster_name,
network_publish_host=elasticsearch_network_publish_host,
multicast_enabled=multicast_enabled,
index_store_type=elasticsearch_index_store_type)
elasticsearch_executor = HTTPExecutor(
command_exec,
'http://{host}:{port}'.format(host=elasticsearch_host,
port=elasticsearch_port),
timeout=60,
)
elasticsearch_executor.start()
def finalize_elasticsearch():
elasticsearch_executor.stop()
shutil.rmtree(home_path)
request.addfinalizer(finalize_elasticsearch)
return elasticsearch_executor
