def running_server():
"""A running fluffy server.
Starts an app server on one port, and an http.server on another port to
serve the static files (much like pgctl does in dev).
"""
tempdir = tempfile.mkdtemp()
os.mkdir(os.path.join(tempdir, 'object'))
os.mkdir(os.path.join(tempdir, 'html'))
app_port = ephemeral_port_reserve.reserve()
static_port = ephemeral_port_reserve.reserve()
settings_path = os.path.join(tempdir, 'settings.py')
with open(settings_path, 'w') as f:
f.write(_templated_config(tempdir, app_port, static_port))
os.environ['FLUFFY_SETTINGS'] = settings_path
app_server = subprocess.Popen(
(
sys.executable,
'-m', 'gunicorn.app.wsgiapp',
'-b', '127.0.0.1:{}'.format(app_port),
'fluffy.run:app',
),
env={
'COVERAGE_PROCESS_START': os.environ.get('COVERAGE_PROCESS_START', ''),
'FLUFFY_SETTINGS': settings_path,
},
)
static_server = subprocess.Popen(
(
sys.executable,
'-m', 'http.server',
'--bind', '127.0.0.1',
str(static_port),
),
cwd=tempdir,
)
_wait_for_http('http://localhost:{}'.format(app_port))
_wait_for_http('http://localhost:{}'.format(static_port))
yield {
'home': 'http://localhost:{}'.format(app_port),
}
time.sleep(1)
static_server.send_signal(signal.SIGTERM)
assert static_server.wait() == -signal.SIGTERM, static_server.returncode
app_server.send_signal(signal.SIGTERM)
assert app_server.wait() == 0, app_server.returncode
shutil.rmtree(tempdir)
def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None):
TailableProc.__init__(self, bitcoin_dir, verbose=False)
if rpcport is None:
rpcport = reserve()
self.bitcoin_dir = bitcoin_dir
self.rpcport = rpcport
self.prefix = 'bitcoind'
regtestdir = os.path.join(bitcoin_dir, 'regtest')
if not os.path.exists(regtestdir):
os.makedirs(regtestdir)
self.cmd_line = [
'bitcoind',
'-datadir={}'.format(bitcoin_dir),
'-printtoconsole',
'-server',
'-logtimestamps',
'-nolisten',
]
# For up to and including 0.16.1, this needs to be in main section.
BITCOIND_CONFIG['rpcport'] = rpcport
# For after 0.16.1 (eg. 3f398d7a17f136cd4a67998406ca41a124ae2966), this
# needs its own [regtest] section.
BITCOIND_REGTEST = {'rpcport': rpcport}
btc_conf_file = os.path.join(bitcoin_dir, 'bitcoin.conf')
write_config(btc_conf_file, BITCOIND_CONFIG, BITCOIND_REGTEST)
self.rpc = SimpleBitcoinProxy(btc_conf_file=btc_conf_file)
self.proxies = []
def __init__(
self,
root_dir,
bitcoind,
executor,
postgres_user,
postgres_pass,
postgres_host="localhost",
):
self.root_dir = root_dir
self.bitcoind = bitcoind
self.daemons = []
self.executor = executor
self.postgres_user = postgres_user
self.postgres_pass = postgres_pass
self.postgres_host = postgres_host
self.coordinator_port = reserve()
self.stk_wallets = []
self.stkman_wallets = []
self.man_wallets = []
self.csv = None
self.emergency_address = None
self.bitcoind_proxy = None
def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None):
TailableProc.__init__(self, bitcoin_dir, verbose=False)
if rpcport is None:
rpcport = reserve()
self.bitcoin_dir = bitcoin_dir
self.rpcport = rpcport
self.prefix = 'bitcoind'
regtestdir = os.path.join(bitcoin_dir, 'regtest')
if not os.path.exists(regtestdir):
os.makedirs(regtestdir)
self.cmd_line = [
'bitcoind',
'-datadir={}'.format(bitcoin_dir),
'-printtoconsole',
'-server',
'-logtimestamps',
'-nolisten',
]
# For up to and including 0.16.1, this needs to be in main section.
BITCOIND_CONFIG['rpcport'] = rpcport
# For after 0.16.1 (eg. 3f398d7a17f136cd4a67998406ca41a124ae2966), this
# needs its own [regtest] section.
BITCOIND_REGTEST = {'rpcport': rpcport}
btc_conf_file = os.path.join(bitcoin_dir, 'bitcoin.conf')
write_config(btc_conf_file, BITCOIND_CONFIG, BITCOIND_REGTEST)
self.rpc = SimpleBitcoinProxy(btc_conf_file=btc_conf_file)
self.proxies = []
def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None):
TailableProc.__init__(self, bitcoin_dir, verbose=False)
if rpcport is None:
rpcport = reserve()
self.bitcoin_dir = bitcoin_dir
self.rpcport = rpcport
self.prefix = 'bitcoind'
regtestdir = os.path.join(bitcoin_dir, 'regtest')
if not os.path.exists(regtestdir):
os.makedirs(regtestdir)
self.cmd_line = [
'bitcoind',
'-datadir={}'.format(bitcoin_dir),
'-printtoconsole',
'-server',
'-regtest',
'-logtimestamps',
'-nolisten',
]
BITCOIND_CONFIG['rpcport'] = rpcport
btc_conf_file = os.path.join(regtestdir, 'bitcoin.conf')
write_config(os.path.join(bitcoin_dir, 'bitcoin.conf'), BITCOIND_CONFIG)
write_config(btc_conf_file, BITCOIND_CONFIG)
self.rpc = SimpleBitcoinProxy(btc_conf_file=btc_conf_file)
def swagger_http_server(self):
def wait_unit_service_starts(url, max_wait_time=10):
start = time.time()
check_url = '{url}/swagger.json'.format(url=url)
while time.time() < start + max_wait_time:
try:
requests.get(check_url, timeout=1)
except requests.ConnectionError:
time.sleep(0.1)
else:
return
port = ephemeral_port_reserve.reserve()
web_service_process = Process(
target=run_bottle_server,
kwargs={'port': port},
)
try:
web_service_process.start()
server_address = 'http://localhost:{port}'.format(port=port)
wait_unit_service_starts(server_address, 10)
yield server_address
finally:
web_service_process.terminate()
def __init__(self, config: Any):
super().__init__(config)
self.cleanup_callbacks: List[Callable[[], None]] = []
self.fundchannel_future: Optional[Any] = None
self.is_fundchannel_kill = False
directory = tempfile.mkdtemp(prefix='lnprototest-clightning-')
self.bitcoind = Bitcoind(directory)
self.bitcoind.start()
self.executor = futures.ThreadPoolExecutor(max_workers=20)
self.lightning_dir = os.path.join(directory, "lightningd")
if not os.path.exists(self.lightning_dir):
os.makedirs(self.lightning_dir)
self.lightning_port = reserve()
self.startup_flags = []
for flag in config.getoption("runner_args"):
self.startup_flags.append("--{}".format(flag))
opts = subprocess.run([
'{}/lightningd/lightningd'.format(LIGHTNING_SRC),
'--list-features-only'
],
stdout=subprocess.PIPE,
check=True).stdout.decode('utf-8').splitlines()
self.options: Dict[str, str] = {}
for o in opts:
k, v = o.split('/')
self.options[k] = v
def bitcoind(directory):
proxyport = reserve()
btc = ProxiedBitcoinD(bitcoin_dir=os.path.join(directory, "bitcoind"),
proxyport=proxyport)
btc.start()
bch_info = btc.rpc.getblockchaininfo()
w_info = btc.rpc.getwalletinfo()
# Make sure we have segwit and some funds
if bch_info['blocks'] < 120:
logging.debug("SegWit not active, generating some more blocks")
btc.rpc.generate(120 - bch_info['blocks'])
elif w_info['balance'] < 1:
logging.debug("Insufficient balance, generating 1 block")
btc.rpc.generate(1)
# Mock `estimatesmartfee` to make c-lightning happy
def mock_estimatesmartfee(r):
return {
"id": r['id'],
"error": None,
"result": {
"feerate": 0.00100001,
"blocks": r['params'][0]
}
}
btc.mock_rpc('estimatesmartfee', mock_estimatesmartfee)
yield btc
try:
btc.rpc.stop()
except Exception:
btc.proc.kill()
btc.proc.wait()
def indexserver(tmpdir_factory):
pypi = tmpdir_factory.mktemp('indexserver').ensure_dir()
_wheel('pip', pypi.strpath, ('venv-update', 'pip-custom-platform'))
_wheel('pip', pypi.strpath, ('venv-update==3.0.0', 'mccabe==0.6.0'))
for pip in ('pip', 'pip_custom_platform.main'):
for pkg in (_testing('cmod_v1'), _testing('cmod_v2')):
_wheel(pip, pypi.strpath, (pkg, ))
port = ephemeral_port_reserve.reserve('0.0.0.0')
index_url = 'http://localhost:{}/simple'.format(port)
proc = subprocess.Popen((
sys.executable,
'-m',
'pypiserver',
'-p',
str(port),
pypi.strpath,
))
try:
timeout = 10
timeout_time = time.time() + timeout
while time.time() < timeout_time:
try:
response = six.moves.urllib.request.urlopen(index_url)
assert response.getcode() == 200
break
except Exception:
print('not up yet')
time.sleep(.1)
else:
raise AssertionError('No pypi after {} seconds'.format(timeout))
yield index_url
finally:
proc.terminate()
def swagger_http_server(self):
def wait_unit_service_starts(url, max_wait_time=10):
start = time.time()
check_url = '{url}/swagger.json'.format(url=url)
while time.time() < start + max_wait_time:
try:
requests.get(check_url, timeout=1)
except requests.ConnectionError:
time.sleep(0.1)
else:
return
port = ephemeral_port_reserve.reserve()
web_service_process = Process(
target=run_bottle_server,
kwargs={'port': port},
)
try:
web_service_process.start()
server_address = 'http://localhost:{port}'.format(port=port)
wait_unit_service_starts(server_address, 10)
yield server_address
finally:
web_service_process.terminate()
def __init__(self, lightning_dir, bitcoind, port, node_id):
super().__init__(lightning_dir, 'lnd({})'.format(node_id))
self.lightning_dir = lightning_dir
self.bitcoind = bitcoind
self.port = port
self.rpc_port = str(reserve())
self.rest_port = str(reserve())
self.prefix = f'lnd-{node_id}'
self.invoice_rpc_active = False
try:
if os.environ['TRAVIS_BUILD_DIR']:
self.tlscertpath = os.environ[
'TRAVIS_BUILD_DIR'] + '/tests/test_utils/test-tls.cert'
except KeyError:
self.tlscertpath = 'test_utils/test-tls.cert'
try:
if os.environ['TRAVIS_BUILD_DIR']:
self.tlskeypath = os.environ[
'TRAVIS_BUILD_DIR'] + '/tests/test_utils/test-tls.key'
except KeyError:
self.tlskeypath = 'test_utils/test-tls.key'
self.cmd_line = [
'lnd', '--bitcoin.active', '--bitcoin.regtest',
'--datadir={}'.format(lightning_dir), '--debuglevel=trace',
'--rpclisten=127.0.0.1:{}'.format(self.rpc_port),
'--restlisten=127.0.0.1:{}'.format(
self.rest_port), '--listen=127.0.0.1:{}'.format(self.port),
'--tlscertpath={}'.format(self.tlscertpath),
'--tlskeypath={}'.format(self.tlskeypath),
'--bitcoin.node=bitcoind',
'--bitcoind.rpchost=127.0.0.1:{}'.format(
BITCOIND_CONFIG.get('rpcport', 18332)),
'--bitcoind.rpcuser=rpcuser', '--bitcoind.rpcpass=rpcpass',
'--bitcoind.zmqpubrawblock=tcp://127.0.0.1:{}'.format(
self.bitcoind.zmqpubrawblock_port),
'--bitcoind.zmqpubrawtx=tcp://127.0.0.1:{}'.format(
self.bitcoind.zmqpubrawtx_port), '--configfile={}'.format(
os.path.join(lightning_dir, self.CONF_NAME)),
'--nobootstrap', '--noseedbackup', '--trickledelay=500'
]
if not os.path.exists(lightning_dir):
os.makedirs(lightning_dir)
with open(os.path.join(lightning_dir, self.CONF_NAME), "w") as f:
f.write("""[Application Options]\n""")
def pick_random_port(hash_key):
"""Return a random port.
Tries to return the same port for the same service each time, when
possible.
"""
hash_number = int(hashlib.sha1(hash_key).hexdigest(), 16)
preferred_port = 33000 + (hash_number % 25000)
return ephemeral_port_reserve.reserve('0.0.0.0', preferred_port)
def test_donation_server(node_factory):
pluginopt = {'plugin': plugin_path, 'allow_warning': True}
l1, l2 = node_factory.line_graph(2, opts=pluginopt)
port = reserve()
l1.rpc.donationserver('start', port)
l1.daemon.wait_for_logs('plugin-donations.py: Process server on port')
msg = l1.rpc.donationserver("stop")
assert msg.startswith(f'stopped server on port')
def threaded_http_server():
port = ephemeral_port_reserve.reserve()
thread = threading.Thread(
target=bottle.run, kwargs={'host': 'localhost', 'port': port},
)
thread.daemon = True
thread.start()
wait_unit_service_starts('http://localhost:{port}'.format(port=port))
yield port
def threaded_http_server():
port = ephemeral_port_reserve.reserve()
thread = threading.Thread(
target=bottle.run, kwargs={'host': 'localhost', 'port': port},
)
thread.daemon = True
thread.start()
wait_unit_service_starts('http://localhost:{port}'.format(port=port))
yield port
def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None):
super().__init__(bitcoin_dir, "bitcoind")
if rpcport is None:
rpcport = reserve()
self.bitcoin_dir = bitcoin_dir
self.prefix = "bitcoind"
BITCOIND_CONFIG["rpcport"] = rpcport
self.rpcport = rpcport
self.zmqpubrawblock_port = reserve()
self.zmqpubrawtx_port = reserve()
regtestdir = os.path.join(bitcoin_dir, "regtest")
if not os.path.exists(regtestdir):
os.makedirs(regtestdir)
conf_file = os.path.join(bitcoin_dir, self.CONF_NAME)
self.cmd_line = [
"bitcoind",
"-datadir={}".format(bitcoin_dir),
"-conf={}".format(conf_file),
"-regtest",
"-logtimestamps",
"-rpcport={}".format(rpcport),
"-printtoconsole=1"
"-debug",
"-rpcuser=rpcuser",
"-rpcpassword=rpcpass",
"-zmqpubrawblock=tcp://127.0.0.1:{}".format(
self.zmqpubrawblock_port),
"-zmqpubrawtx=tcp://127.0.0.1:{}".format(self.zmqpubrawtx_port),
# "-zmqpubrawblockhwm=0",
# "-zmqpubrawtxhwm=0",
]
BITCOIND_CONFIG["rpcport"] = rpcport
write_config(os.path.join(bitcoin_dir, self.CONF_NAME),
BITCOIND_CONFIG)
write_config(os.path.join(regtestdir, self.CONF_NAME), BITCOIND_CONFIG)
self.rpc = BitcoinRpc(rpcport=rpcport,
rpcuser="******",
rpcpassword="******")
def test_grpc_wrong_auth(node_factory):
"""An mTLS client certificate should only be usable with its node
We create two instances, each generates its own certs and keys,
and then we try to cross the wires.
"""
# These only exist if we have rust!
from node_pb2_grpc import NodeStub # noqa: E402
import node_pb2 as nodepb # noqa: E402
grpc_port = reserve()
bin_path = Path.cwd() / "target" / "debug" / "cln-grpc"
l1, l2 = node_factory.get_nodes(2,
opts={
"plugin": str(bin_path),
"start": False,
"grpc-port": str(grpc_port),
})
l1.start()
wait_for_grpc_start(l1)
def connect(node):
p = Path(node.daemon.lightning_dir) / TEST_NETWORK
cert, key, ca = [
f.open('rb').read()
for f in [p / 'client.pem', p / 'client-key.pem', p / "ca.pem"]
]
creds = grpc.ssl_channel_credentials(
root_certificates=ca,
private_key=key,
certificate_chain=cert,
)
channel = grpc.secure_channel(
f"localhost:{grpc_port}",
creds,
options=(('grpc.ssl_target_name_override', 'cln'), ))
return NodeStub(channel)
stub = connect(l1)
# This should work, it's the correct node
stub.Getinfo(nodepb.GetinfoRequest())
l1.stop()
l2.start()
wait_for_grpc_start(l2)
# This should not work, it's a different node
with pytest.raises(Exception,
match=r'Socket closed|StatusCode.UNAVAILABLE'):
stub.Getinfo(nodepb.GetinfoRequest())
# Now load the correct ones and we should be good to go
stub = connect(l2)
stub.Getinfo(nodepb.GetinfoRequest())
def revaultd_stakeholder(bitcoind, directory):
datadir = os.path.join(directory, "revaultd")
os.makedirs(datadir, exist_ok=True)
(stks, cosigs, mans, _, _, _) = get_participants(2, 3)
cpfp_xprivs = [
bytes.fromhex(
"0435839400000000000000000060499f801b896d83179a4374aeb7822aaeaceaa0db1f85ee3e904c4defbd9689004b03d6fc340455b363f51020ad3ecca4f0850280cf436c70c727923f6db46c3e",
)
]
cpfp_xpubs = [
"tpubD6NzVbkrYhZ4XJDrzRvuxHEyQaPd1mwwdDofEJwekX18tAdsqeKfxss79AJzg1431FybXg5rfpTrJF4iAhyR7RubberdzEQXiRmXGADH2eA"
]
stks_xpubs = [stk.get_xpub() for stk in stks]
cosigs_keys = []
mans_xpubs = [man.get_xpub() for man in mans]
(dep_desc, unv_desc, cpfp_desc) = get_descriptors(stks_xpubs, cosigs_keys,
mans_xpubs,
len(mans_xpubs),
cpfp_xpubs, 232)
stk_config = {
"keychain":
stks[0],
"watchtowers": [{
"host": "127.0.0.1:1",
"noise_key": os.urandom(32).hex()
}],
# We use a dummy one since we don't use it anyways
"emergency_address":
"bcrt1qewc2348370pgw8kjz8gy09z8xyh0d9fxde6nzamd3txc9gkmjqmq8m4cdq",
}
coordinator_noise_key = (
"d91563973102454a7830137e92d0548bc83b4ea2799f1df04622ca1307381402")
bitcoind_cookie = os.path.join(bitcoind.bitcoin_dir, "regtest", ".cookie")
revaultd = StakeholderRevaultd(
datadir,
dep_desc,
unv_desc,
cpfp_desc,
os.urandom(32),
coordinator_noise_key,
reserve(),
bitcoind.rpcport,
bitcoind_cookie,
stk_config=stk_config,
wt_process=None,
)
try:
revaultd.start()
yield revaultd
except Exception:
revaultd.cleanup()
raise
revaultd.cleanup()
def test_grpc_connect(node_factory):
"""Attempts to connect to the grpc interface and call getinfo"""
# These only exist if we have rust!
from node_pb2_grpc import NodeStub # noqa: E402
import node_pb2 as nodepb # noqa: E402
from primitives_pb2 import AmountOrAny, Amount # noqa: E402
grpc_port = reserve()
bin_path = Path.cwd() / "target" / "debug" / "cln-grpc"
l1 = node_factory.get_node(options={
"plugin": str(bin_path),
"grpc-port": str(grpc_port)
})
p = Path(l1.daemon.lightning_dir) / TEST_NETWORK
cert_path = p / "client.pem"
key_path = p / "client-key.pem"
ca_cert_path = p / "ca.pem"
creds = grpc.ssl_channel_credentials(
root_certificates=ca_cert_path.open('rb').read(),
private_key=key_path.open('rb').read(),
certificate_chain=cert_path.open('rb').read())
wait_for_grpc_start(l1)
channel = grpc.secure_channel(f"localhost:{grpc_port}",
creds,
options=(('grpc.ssl_target_name_override',
'cln'), ))
stub = NodeStub(channel)
response = stub.Getinfo(nodepb.GetinfoRequest())
print(response)
response = stub.ListFunds(nodepb.ListfundsRequest())
print(response)
inv = stub.Invoice(
nodepb.InvoiceRequest(msatoshi=AmountOrAny(any=True),
description="hello",
label="lbl1",
preimage=b"\x00" * 32,
cltv=24))
print(inv)
rates = stub.Feerates(nodepb.FeeratesRequest(style='PERKB'))
print(rates)
# Test a failing RPC call, so we know that errors are returned correctly.
with pytest.raises(Exception, match=r'Duplicate label'):
# This request creates a label collision
stub.Invoice(
nodepb.InvoiceRequest(
msatoshi=AmountOrAny(amount=Amount(msat=12345)),
description="hello",
label="lbl1",
))
def pick_random_port(service_name):
"""Return a random port.
Tries to return the same port for the same service each time, when
possible.
"""
hash_key = f"{service_name},{getpass.getuser()}".encode("utf8")
hash_number = int(hashlib.sha1(hash_key).hexdigest(), 16)
preferred_port = 33000 + (hash_number % 25000)
return ephemeral_port_reserve.reserve("0.0.0.0", preferred_port)
def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None):
super().__init__(bitcoin_dir, 'bitcoind')
if rpcport is None:
rpcport = reserve()
self.bitcoin_dir = bitcoin_dir
self.prefix = 'bitcoind'
BITCOIND_CONFIG['rpcport'] = rpcport
self.rpcport = rpcport
self.zmqpubrawblock_port = reserve()
self.zmqpubrawtx_port = reserve()
regtestdir = os.path.join(bitcoin_dir, 'regtest')
if not os.path.exists(regtestdir):
os.makedirs(regtestdir)
conf_file = os.path.join(bitcoin_dir, self.CONF_NAME)
self.cmd_line = [
'bitcoind',
'-datadir={}'.format(bitcoin_dir),
'-conf={}'.format(conf_file),
'-regtest',
'-logtimestamps',
'-rpcport={}'.format(rpcport),
'-printtoconsole=1'
'-debug',
'-rpcuser=rpcuser',
'-rpcpassword=rpcpass',
'-zmqpubrawblock=tcp://127.0.0.1:{}'.format(
self.zmqpubrawblock_port),
'-zmqpubrawtx=tcp://127.0.0.1:{}'.format(self.zmqpubrawtx_port),
]
BITCOIND_CONFIG['rpcport'] = rpcport
write_config(os.path.join(bitcoin_dir, self.CONF_NAME),
BITCOIND_CONFIG)
write_config(os.path.join(regtestdir, self.CONF_NAME), BITCOIND_CONFIG)
self.rpc = BitcoinRpc(rpcport=rpcport,
rpcuser='******',
rpcpassword='******')
def assert_ip(ip):
port = reserve(ip)
# show that we can't bind to it without SO_REUSEADDR
error = bind_naive(ip, port)
assert error and error.errno == errno.EADDRINUSE, error
# show that we *can* bind to it without SO_REUSEADDR, after release
sock = bind_reuse(ip, port)
sname = sock.getsockname()
assert sname == (ip, port), (sname, port)
def start(self):
passfile = os.path.join(self.directory, "pgpass.txt")
# Need to write a tiny file containing the password so `initdb` can
# pick it up
with open(passfile, 'w') as f:
f.write('cltest\n')
# Look for a postgres directory that isn't taken yet. Not locking
# since this is run in a single-threaded context, at the start of each
# test. Multiple workers have separate directories, so they can't
# trample each other either.
for i in itertools.count():
self.pgdir = os.path.join(self.directory, 'pgsql-{}'.format(i))
if not os.path.exists(self.pgdir):
break
initdb, postgres = self.locate_path()
subprocess.check_call([
initdb,
'--pwfile={}'.format(passfile),
'--pgdata={}'.format(self.pgdir),
'--auth=trust',
'--username=postgres',
])
conffile = os.path.join(self.pgdir, 'postgresql.conf')
with open(conffile, 'a') as f:
f.write('max_connections = 1000\nshared_buffers = 240MB\n')
self.port = reserve()
self.proc = subprocess.Popen([
postgres,
'-k',
'/tmp/', # So we don't use /var/lib/...
'-D',
self.pgdir,
'-p',
str(self.port),
'-F',
'-i',
])
# Hacky but seems to work ok (might want to make the postgres proc a
# TailableProc as well if too flaky).
for i in range(30):
try:
self.conn = psycopg2.connect(
"dbname=template1 user=postgres host=localhost port={}".
format(self.port))
break
except Exception:
time.sleep(0.5)
# Required for CREATE DATABASE to work
self.conn.set_isolation_level(
psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
def dummy_deb_nginx():
image_name = _build_testing_image('dummy_deb_nginx')
port = reserve()
dummy_deb_nginx_name = _get_name_with_random_suffix('dummy_deb_nginx')
run_dummy_deb_nginx_command = ('docker', 'run', '-d', '-p',
f'127.0.0.1:{port}:80', '--rm', '--name',
dummy_deb_nginx_name, image_name)
subprocess.check_call(run_dummy_deb_nginx_command)
image_ip = inspect_image(dummy_deb_nginx_name)['NetworkSettings'][
'Networks']['bridge']['IPAddress']
yield (dummy_deb_nginx_name, image_ip)
_delete_image(image_name)
def __init__(self, bitcoin_dir, rpcport=None):
TailableProc.__init__(self, bitcoin_dir, verbose=False)
if rpcport is None:
rpcport = reserve()
self.bitcoin_dir = bitcoin_dir
self.rpcport = rpcport
self.p2pport = reserve()
self.prefix = "bitcoind"
regtestdir = os.path.join(bitcoin_dir, "regtest")
if not os.path.exists(regtestdir):
os.makedirs(regtestdir)
self.cmd_line = [
"bitcoind",
"-datadir={}".format(bitcoin_dir),
"-printtoconsole",
"-server",
"-logtimestamps",
"-rpcthreads=4",
]
bitcoind_conf = {
"port": self.p2pport,
"rpcport": rpcport,
"debug": 1,
"fallbackfee": Decimal(1000) / bitcoin.core.COIN,
}
self.conf_file = os.path.join(bitcoin_dir, "bitcoin.conf")
with open(self.conf_file, "w") as f:
f.write("chain=regtest\n")
f.write("[regtest]\n")
for k, v in bitcoind_conf.items():
f.write(f"{k}={v}\n")
self.rpc = SimpleBitcoinProxy(
bitcoind_dir=self.bitcoin_dir, bitcoind_port=self.rpcport
)
self.proxies = []
def __init__(self, bitcoin_dir, rpcport=None):
TailableProc.__init__(self, bitcoin_dir, verbose=False)
if rpcport is None:
rpcport = reserve()
self.bitcoin_dir = bitcoin_dir
self.rpcport = rpcport
self.p2pport = reserve()
self.prefix = 'bitcoind'
regtestdir = os.path.join(bitcoin_dir, 'regtest')
if not os.path.exists(regtestdir):
os.makedirs(regtestdir)
self.cmd_line = [
'bitcoind',
'-datadir={}'.format(bitcoin_dir),
'-printtoconsole',
'-server',
'-logtimestamps',
'-rpcthreads=4',
]
bitcoind_conf = {
'port': self.p2pport,
'rpcport': rpcport,
'debug': 1,
'fallbackfee': Decimal(1000) / bitcoin.core.COIN,
}
self.conf_file = os.path.join(bitcoin_dir, 'bitcoin.conf')
with open(self.conf_file, 'w') as f:
f.write(f"chain=regtest\n")
f.write(f"[regtest]\n")
for k, v in bitcoind_conf.items():
f.write(f"{k}={v}\n")
self.rpc = SimpleBitcoinProxy(bitcoind_dir=self.bitcoin_dir,
bitcoind_port=self.rpcport)
self.proxies = []
def revaultd_manager(bitcoind, directory):
datadir = os.path.join(directory, "revaultd")
os.makedirs(datadir, exist_ok=True)
(stks, cosigs, mans, _, _, _) = get_participants(2, 3)
cpfp_seed = os.urandom(32)
cpfp_xprivs = [bip32.BIP32.from_seed(cpfp_seed, network="test")]
cpfp_xpubs = [cpfp_xprivs[0].get_xpub()]
stks_xpubs = [stk.get_xpub() for stk in stks]
cosigs_keys = []
mans_xpubs = [man.get_xpub() for man in mans]
(dep_desc, unv_desc, cpfp_desc) = get_descriptors(stks_xpubs, cosigs_keys,
mans_xpubs,
len(mans_xpubs),
cpfp_xpubs, 232)
man_config = {
"keychain":
mans[0],
"cosigners": [{
"host": "127.0.0.1:1",
"noise_key": os.urandom(32)
}],
# We use a dummy one since we don't use it anyways
"emergency_address":
"bcrt1qewc2348370pgw8kjz8gy09z8xyh0d9fxde6nzamd3txc9gkmjqmq8m4cdq",
}
coordinator_noise_key = (
"d91563973102454a7830137e92d0548bc83b4ea2799f1df04622ca1307381402")
bitcoind_cookie = os.path.join(bitcoind.bitcoin_dir, "regtest", ".cookie")
revaultd = ManagerRevaultd(
datadir,
dep_desc,
unv_desc,
cpfp_desc,
os.urandom(32),
coordinator_noise_key,
reserve(),
bitcoind.rpcport,
bitcoind_cookie,
man_config=man_config,
cpfp_seed=cpfp_seed,
)
try:
revaultd.start()
yield revaultd
except Exception:
revaultd.cleanup()
raise
revaultd.cleanup()
def __init__(self, lightning_dir, bitcoind, port):
TailableProc.__init__(self, lightning_dir, "eclair({})".format(port))
self.lightning_dir = lightning_dir
self.bitcoind = bitcoind
self.port = port
self.rpc_port = str(reserve())
self.prefix = 'eclair'
self.cmd_line = [
'java', '-Declair.datadir={}'.format(lightning_dir),
'-Dlogback.configurationFile={}'.format(
os.path.join(lightning_dir,
'logback.xml')), '-jar', 'bin/eclair.jar'
]
if not os.path.exists(lightning_dir):
os.makedirs(lightning_dir)
shutil.copyfile('logback.xml',
os.path.join(lightning_dir, "logback.xml"))
# Adapt the config and store it
with open('src/eclair/eclair-core/src/main/resources/reference.conf'
) as f:
config = f.read()
replacements = [
('"testnet"', '"regtest"'),
('enabled = false // disabled by default for security reasons',
'enabled = true'),
('password = ""', 'password = "******"'),
('9735', str(port)),
('18332', str(self.bitcoind.rpcport)),
('8080', str(self.rpc_port)),
('"test"', '"regtest"'),
('"foo"', '"rpcuser"'),
('"bar"', '"rpcpass"'),
('zmqblock = "tcp://127.0.0.1:29000"',
'zmqblock = "tcp://127.0.0.1:{}"'.format(
self.bitcoind.zmqpubrawblock_port)),
('zmqtx = "tcp://127.0.0.1:29000"',
'zmqtx = "tcp://127.0.0.1:{}"'.format(
self.bitcoind.zmqpubrawtx_port)),
('use-old-api = false', 'use-old-api = true'),
]
for old, new in replacements:
config = config.replace(old, new)
with open(os.path.join(lightning_dir, "eclair.conf"), "w") as f:
f.write(config)
def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None):
TailableProc.__init__(self, bitcoin_dir, verbose=False)
if rpcport is None:
rpcport = reserve()
self.bitcoin_dir = bitcoin_dir
self.rpcport = rpcport
self.p2pport = reserve()
self.prefix = 'bitcoind'
regtestdir = os.path.join(bitcoin_dir, 'regtest')
if not os.path.exists(regtestdir):
os.makedirs(regtestdir)
self.cmd_line = [
'bitcoind',
'-datadir={}'.format(bitcoin_dir),
'-printtoconsole',
'-server',
'-logtimestamps',
'-addresstype=bech32',
'-rpcthreads=4',
]
BITCOIND_REGTEST = {
'port': self.p2pport,
'rpcport': rpcport,
'debug': 1,
'fallbackfee': Decimal(1000) / bitcoin.core.COIN,
}
self.conf_file = os.path.join(bitcoin_dir, 'bitcoin.conf')
write_config(self.conf_file, BITCOIND_REGTEST)
self.rpc = SimpleBitcoinProxy(btc_conf_file=self.conf_file)
self.proxies = []
# So that it can locate the cookie file
bitcoin.SelectParams("regtest")
def start(self):
self.logger.info("start")
self.tcp_port = reserve()
self.rpc_port = reserve()
os.environ.update({
"COIN":
"BitcoinSegwit",
"TCP_PORT":
str(self.tcp_port),
"RPC_PORT":
str(self.rpc_port),
"NET":
"regtest",
"DAEMON_URL":
"http://*****:*****@127.0.0.1:" + str(self.bitcoind.rpcport),
"DB_DIRECTORY":
self.directory,
"MAX_SESSIONS":
"50",
})
def target():
loop = self.loop
asyncio.set_event_loop(loop)
env = Env()
env.loop_policy = asyncio.get_event_loop_policy()
self.controller = Controller(env)
logging.basicConfig(level=logging.DEBUG)
loop.run_until_complete(
asyncio.wait(
[self.controller.serve(self.evt),
self.evt.wait()],
return_when=asyncio.FIRST_COMPLETED))
loop.close()
self.thread = threading.Thread(target=target)
self.thread.start()
def __init__(self, lightning_dir, bitcoind, port):
super().__init__(lightning_dir, 'lnd({})'.format(port))
self.lightning_dir = lightning_dir
self.bitcoind = bitcoind
self.port = port
self.rpc_port = str(reserve())
self.rest_port = str(reserve())
self.prefix = 'lnd'
self.cmd_line = [
'bin/lnd',
'--bitcoin.active',
'--bitcoin.regtest',
'--datadir={}'.format(lightning_dir),
'--debuglevel=trace',
'--rpclisten=127.0.0.1:{}'.format(self.rpc_port),
'--restlisten=127.0.0.1:{}'.format(self.rest_port),
'--listen=127.0.0.1:{}'.format(self.port),
'--tlscertpath=tls.cert',
'--tlskeypath=tls.key',
'--bitcoin.node=bitcoind',
'--bitcoind.rpchost=127.0.0.1:{}'.format(BITCOIND_CONFIG.get('rpcport', 18332)),
'--bitcoind.rpcuser=rpcuser',
'--bitcoind.rpcpass=rpcpass',
'--bitcoind.zmqpubrawblock=tcp://127.0.0.1:{}'.format(self.bitcoind.zmqpubrawblock_port),
'--bitcoind.zmqpubrawtx=tcp://127.0.0.1:{}'.format(self.bitcoind.zmqpubrawtx_port),
'--configfile={}'.format(os.path.join(lightning_dir, self.CONF_NAME)),
'--no-macaroons',
'--nobootstrap',
'--noseedbackup',
'--trickledelay=500'
]
if not os.path.exists(lightning_dir):
os.makedirs(lightning_dir)
with open(os.path.join(lightning_dir, self.CONF_NAME), "w") as f:
f.write("""[Application Options]\n""")
def __init__(self, bitcoin_dir, rpcport=None):
TailableProc.__init__(self, bitcoin_dir, verbose=False)
if rpcport is None:
rpcport = reserve()
self.bitcoin_dir = bitcoin_dir
self.rpcport = rpcport
self.p2pport = reserve()
self.prefix = "bitcoind"
regtestdir = os.path.join(bitcoin_dir, "regtest")
if not os.path.exists(regtestdir):
os.makedirs(regtestdir)
self.cmd_line = [
BITCOIND_PATH,
"-datadir={}".format(bitcoin_dir),
"-printtoconsole",
"-server",
]
bitcoind_conf = {
"port": self.p2pport,
"rpcport": rpcport,
"debug": 1,
"fallbackfee": Decimal(1000) / COIN,
"rpcthreads": 32,
}
self.conf_file = os.path.join(bitcoin_dir, "bitcoin.conf")
with open(self.conf_file, "w") as f:
f.write("chain=regtest\n")
f.write("[regtest]\n")
for k, v in bitcoind_conf.items():
f.write(f"{k}={v}\n")
self.rpc = BitcoindRpcInterface(bitcoin_dir, "regtest", rpcport)
def integration_server():
script_name = os.path.join(
os.path.dirname(__file__), "../../testing/integration_server.py"
)
server_port = ephemeral_port_reserve.reserve()
server = subprocess.Popen(
["python", script_name, "-p", str(server_port)],
stdin=None,
stdout=None,
stderr=None,
)
wait_unit_service_starts("http://localhost:{port}".format(port=server_port))
yield "http://localhost:{}".format(server_port)
server.terminate()
def get_node(self, implementation, node_id):
# node_id = self.next_id
# self.next_id += 1
lightning_dir = os.path.join(TEST_DIR, self.testname,
"node-{}/".format(node_id))
port = reserve()
node = implementation(lightning_dir,
port,
self.bitcoind,
executor=self.executor,
node_id=node_id)
self.nodes.append(node)
node.daemon.start()
return node
def not_answering_http_server(self):
yield 'http://localhost:{}'.format(ephemeral_port_reserve.reserve())
def pypi_port():
yield reserve()
def get_next_port(self):
with self.lock:
return reserve()
