def setUp(self):
super().setUp()
self.reactor = self.clock
self.manager = HathorManager(self.reactor, **self._manager_kwargs())
self.manager.allow_mining_without_peers()
self.manager.test_mode = TestMode.TEST_ALL_WEIGHT
self.manager.start()
def test_wrong_stop(self):
manager = HathorManager(self.clock, tx_storage=self.tx_storage)
with self.assertRaises(Exception):
manager.stop()
manager.start()
manager.stop()
with self.assertRaises(Exception):
manager.stop()
def setUp(self, tx_storage, reactor=None):
if not reactor:
self.reactor = Clock()
else:
self.reactor = reactor
self.reactor.advance(time.time())
self.tx_storage = tx_storage
assert tx_storage.first_timestamp > 0
tx_storage._manually_initialize()
self.genesis = self.tx_storage.get_all_genesis()
self.genesis_blocks = [tx for tx in self.genesis if tx.is_block]
self.genesis_txs = [tx for tx in self.genesis if not tx.is_block]
from hathor.manager import HathorManager
self.tmpdir = tempfile.mkdtemp()
wallet = Wallet(directory=self.tmpdir)
wallet.unlock(b'teste')
self.manager = HathorManager(self.reactor, tx_storage=self.tx_storage, wallet=wallet)
self.tx_storage.wallet_index = WalletIndex(self.manager.pubsub)
self.tx_storage.tokens_index = TokensIndex()
block_parents = [tx.hash for tx in chain(self.genesis_blocks, self.genesis_txs)]
output = TxOutput(200, bytes.fromhex('1e393a5ce2ff1c98d4ff6892f2175100f2dad049'))
self.block = Block(timestamp=MIN_TIMESTAMP, weight=12, outputs=[output], parents=block_parents,
nonce=100781, storage=tx_storage)
self.block.resolve()
self.block.verify()
tx_parents = [tx.hash for tx in self.genesis_txs]
tx_input = TxInput(
tx_id=self.genesis_blocks[0].hash, index=0,
data=bytes.fromhex('46304402203470cb9818c9eb842b0c433b7e2b8aded0a51f5903e971649e870763d0266a'
'd2022049b48e09e718c4b66a0f3178ef92e4d60ee333d2d0e25af8868acf5acbb35aaa583'
'056301006072a8648ce3d020106052b8104000a034200042ce7b94cba00b654d4308f8840'
'7345cacb1f1032fb5ac80407b74d56ed82fb36467cb7048f79b90b1cf721de57e942c5748'
'620e78362cf2d908e9057ac235a63'))
self.tx = Transaction(
timestamp=MIN_TIMESTAMP + 2, weight=10, nonce=932049, inputs=[tx_input], outputs=[output],
tokens=[bytes.fromhex('0023be91834c973d6a6ddd1a0ae411807b7c8ef2a015afb5177ee64b666ce602')],
parents=tx_parents, storage=tx_storage)
self.tx.resolve()
# Disable weakref to test the internal methods. Otherwise, most methods return objects from weakref.
self.tx_storage._disable_weakref()
self.tx_storage.enable_lock()
def test_invalid_arguments(self):
# this is a base case, it shouldn't raise any error
# (otherwise we might not be testing the correct thing below)
manager = HathorManager(self.clock, tx_storage=self.tx_storage)
del manager
# disabling both sync versions should be invalid
with self.assertRaises(TypeError):
HathorManager(self.clock,
tx_storage=self.tx_storage,
enable_sync_v1=False,
enable_sync_v2=False)
# not passing a storage should be invalid
with self.assertRaises(TypeError):
HathorManager(self.clock)
def gen_new_double_spending(manager: HathorManager, *, use_same_parents: bool = False) -> Transaction:
tx_interval = random.choice(list(manager.tx_storage.get_tx_tips()))
tx = manager.tx_storage.get_transaction(tx_interval.data)
txin = random.choice(tx.inputs)
from hathor.transaction.scripts import P2PKH, parse_address_script
spent_tx = tx.get_spent_tx(txin)
spent_txout = spent_tx.outputs[txin.index]
p2pkh = parse_address_script(spent_txout.script)
assert isinstance(p2pkh, P2PKH)
from hathor.wallet.base_wallet import WalletInputInfo, WalletOutputInfo
value = spent_txout.value
private_key = manager.wallet.get_private_key(p2pkh.address)
inputs = [WalletInputInfo(tx_id=txin.tx_id, index=txin.index, private_key=private_key)]
address = manager.wallet.get_unused_address(mark_as_used=True)
outputs = [WalletOutputInfo(address=decode_address(address), value=int(value), timelock=None)]
tx2 = manager.wallet.prepare_transaction(Transaction, inputs, outputs, manager.tx_storage)
tx2.storage = manager.tx_storage
tx2.weight = 1
tx2.timestamp = max(tx.timestamp + 1, int(manager.reactor.seconds()))
if use_same_parents:
tx2.parents = list(tx.parents)
else:
tx2.parents = manager.get_new_tx_parents(tx2.timestamp)
tx2.resolve()
return tx2
def gen_new_double_spending(manager: HathorManager, *, use_same_parents: bool = False,
tx: Optional[Transaction] = None, weight: float = 1) -> Transaction:
if tx is None:
tx_candidates = manager.get_new_tx_parents()
genesis = manager.tx_storage.get_all_genesis()
genesis_txs = [tx for tx in genesis if not tx.is_block]
# XXX: it isn't possible to double-spend a genesis transaction, thus we remove it from tx_candidates
for genesis_tx in genesis_txs:
if genesis_tx.hash in tx_candidates:
tx_candidates.remove(genesis_tx.hash)
if not tx_candidates:
raise NoCandidatesError()
# assert tx_candidates, 'Must not be empty, otherwise test was wrongly set up'
tx_hash = manager.rng.choice(tx_candidates)
tx = cast(Transaction, manager.tx_storage.get_transaction(tx_hash))
txin = manager.rng.choice(tx.inputs)
from hathor.transaction.scripts import P2PKH, parse_address_script
spent_tx = tx.get_spent_tx(txin)
spent_txout = spent_tx.outputs[txin.index]
p2pkh = parse_address_script(spent_txout.script)
assert isinstance(p2pkh, P2PKH)
from hathor.wallet.base_wallet import WalletInputInfo, WalletOutputInfo
value = spent_txout.value
wallet = manager.wallet
assert wallet is not None
private_key = wallet.get_private_key(p2pkh.address)
inputs = [WalletInputInfo(tx_id=txin.tx_id, index=txin.index, private_key=private_key)]
address = wallet.get_unused_address(mark_as_used=True)
outputs = [WalletOutputInfo(address=decode_address(address), value=int(value), timelock=None)]
tx2 = wallet.prepare_transaction(Transaction, inputs, outputs)
tx2.storage = manager.tx_storage
tx2.weight = weight
tx2.timestamp = max(tx.timestamp + 1, int(manager.reactor.seconds()))
if use_same_parents:
tx2.parents = list(tx.parents)
else:
tx2.parents = manager.get_new_tx_parents(tx2.timestamp)
tx2.resolve()
return tx2
def create_peer(self,
network,
peer_id=None,
wallet=None,
tx_storage=None,
unlock_wallet=True,
wallet_index=False,
capabilities=None):
if peer_id is None:
peer_id = PeerId()
if not wallet:
wallet = self._create_test_wallet()
if unlock_wallet:
wallet.unlock(b'MYPASS')
manager = HathorManager(
self.clock,
peer_id=peer_id,
network=network,
wallet=wallet,
tx_storage=tx_storage,
wallet_index=wallet_index,
capabilities=capabilities,
)
manager.avg_time_between_blocks = 0.0001
manager.test_mode = TestMode.TEST_ALL_WEIGHT
manager._full_verification = True
manager.start()
self.run_to_completion()
return manager
def test_manager_connections(self):
tx_storage = TransactionMemoryStorage()
tmpdir = tempfile.mkdtemp()
wallet = Wallet(directory=tmpdir)
wallet.unlock(b'teste')
manager = HathorManager(self.clock, tx_storage=tx_storage, wallet=wallet)
endpoint = 'tcp://127.0.0.1:8005'
manager.connections.connect_to(endpoint, use_ssl=True)
self.assertFalse(endpoint in manager.connections.connecting_peers)
self.assertFalse(endpoint in manager.connections.handshaking_peers)
self.assertFalse(endpoint in manager.connections.connected_peers)
manager.stop()
manager.stop()
shutil.rmtree(tmpdir)
class _ResourceTest(unittest.TestCase):
def _manager_kwargs(self):
peer_id = PeerId()
network = 'testnet'
wallet = self._create_test_wallet()
tx_storage = getattr(self, 'tx_storage', None)
if tx_storage is None:
if self.use_memory_storage:
from hathor.transaction.storage.memory_storage import TransactionMemoryStorage
tx_storage = TransactionMemoryStorage()
else:
from hathor.transaction.storage.rocksdb_storage import TransactionRocksDBStorage
directory = tempfile.mkdtemp()
self.tmpdirs.append(directory)
tx_storage = TransactionRocksDBStorage(directory)
assert (
hasattr(self, '_enable_sync_v1') and
hasattr(self, '_enable_sync_v2') and
(self._enable_sync_v1 or self._enable_sync_v2)
), (
'Please set both `_enable_sync_v1` and `_enable_sync_v2` on the class. '
'Also they can\'t both be False. '
'This is by design so we don\'t forget to test for multiple sync versions.'
)
return dict(
peer_id=peer_id,
network=network,
wallet=wallet,
tx_storage=tx_storage,
wallet_index=True,
enable_sync_v1=self._enable_sync_v1,
enable_sync_v2=self._enable_sync_v2,
)
def setUp(self):
super().setUp()
self.reactor = self.clock
self.manager = HathorManager(self.reactor, **self._manager_kwargs())
self.manager.allow_mining_without_peers()
_set_test_mode(TestMode.TEST_ALL_WEIGHT)
self.manager.start()
def tearDown(self):
return self.manager.stop()
def gen_custom_tx(manager: HathorManager, tx_inputs: List[Tuple[Transaction, int]], *, n_outputs: int = 1,
base_parent: Optional[Transaction] = None, weight: Optional[float] = None) -> Transaction:
"""Generate a custom tx based on the inputs and outputs. It gives full control to the
inputs and can be used to generate conflicts and specific patterns in the DAG."""
inputs = []
value = 0
parents = []
for tx_base, txout_index in tx_inputs:
assert tx_base.hash is not None
spent_tx = tx_base
spent_txout = spent_tx.outputs[txout_index]
p2pkh = parse_address_script(spent_txout.script)
assert isinstance(p2pkh, P2PKH)
from hathor.wallet.base_wallet import WalletInputInfo, WalletOutputInfo
value += spent_txout.value
wallet = manager.wallet
assert wallet is not None
assert spent_tx.hash is not None
private_key = wallet.get_private_key(p2pkh.address)
inputs.append(WalletInputInfo(tx_id=spent_tx.hash, index=txout_index, private_key=private_key))
if not tx_base.is_block:
parents.append(tx_base.hash)
assert wallet is not None
address = wallet.get_unused_address(mark_as_used=True)
if n_outputs == 1:
outputs = [WalletOutputInfo(address=decode_address(address), value=int(value), timelock=None)]
elif n_outputs == 2:
assert int(value) > 1
outputs = [
WalletOutputInfo(address=decode_address(address), value=int(value) - 1, timelock=None),
WalletOutputInfo(address=decode_address(address), value=1, timelock=None),
]
else:
raise NotImplementedError
tx2 = wallet.prepare_transaction(Transaction, inputs, outputs)
tx2.storage = manager.tx_storage
tx2.timestamp = max(tx_base.timestamp + 1, int(manager.reactor.seconds()))
tx2.parents = parents[:2]
if len(tx2.parents) < 2:
if base_parent:
assert base_parent.hash is not None
tx2.parents.append(base_parent.hash)
elif not tx_base.is_block:
tx2.parents.append(tx_base.parents[0])
else:
tx2.parents.extend(manager.get_new_tx_parents(tx2.timestamp))
tx2.parents = tx2.parents[:2]
assert len(tx2.parents) == 2
tx2.weight = weight or 25
tx2.update_hash()
return tx2
def tests_init_with_stratum(self):
manager = HathorManager(self.clock,
tx_storage=self.tx_storage,
stratum_port=50505)
manager.start()
manager.stop()
del manager
class _ResourceTest(unittest.TestCase):
def _manager_kwargs(self):
peer_id = PeerId()
network = 'testnet'
wallet = self._create_test_wallet()
tx_storage = getattr(self, 'tx_storage', None)
return dict(
peer_id=peer_id,
network=network,
wallet=wallet,
tx_storage=tx_storage,
wallet_index=True
)
def setUp(self):
super().setUp()
self.reactor = self.clock
self.manager = HathorManager(self.reactor, **self._manager_kwargs())
self.manager.allow_mining_without_peers()
self.manager.test_mode = TestMode.TEST_ALL_WEIGHT
self.manager.start()
def tearDown(self):
return self.manager.stop()
def test_manager_connections(self):
tx_storage = TransactionMemoryStorage()
tmpdir = tempfile.mkdtemp()
wallet = Wallet(directory=tmpdir)
wallet.unlock(b'teste')
manager = HathorManager(self.clock,
tx_storage=tx_storage,
wallet=wallet)
endpoint = 'tcp://127.0.0.1:8005'
manager.connections.connect_to(endpoint, use_ssl=True)
self.assertNotIn(endpoint,
manager.connections.iter_not_ready_endpoints())
self.assertNotIn(endpoint,
manager.connections.iter_ready_connections())
self.assertNotIn(endpoint, manager.connections.iter_all_connections())
shutil.rmtree(tmpdir)
def create_peer(self, network: Optional[str] = None, peer_id: Optional[PeerId] = None,
enable_sync_v1: bool = True, enable_sync_v2: bool = True,
soft_voided_tx_ids: Optional[Set[bytes]] = None) -> HathorManager:
assert self._started
if network is None:
network = self._network
wallet = HDWallet(gap_limit=2)
wallet._manually_initialize()
assert peer_id is not None # XXX: temporary, for checking that tests are using the peer_id
if peer_id is None:
peer_id = PeerId()
tx_storage = TransactionMemoryStorage()
manager = HathorManager(
self._clock,
peer_id=peer_id,
network=network,
wallet=wallet,
enable_sync_v1=enable_sync_v1,
enable_sync_v2=enable_sync_v2,
tx_storage=tx_storage,
rng=Random(self.rng.getrandbits(64)),
soft_voided_tx_ids=soft_voided_tx_ids,
)
manager.reactor = self._clock
manager._full_verification = True
manager.start()
self.run_to_completion()
# Don't use it anywhere else. It is unsafe to generate mnemonic words like this.
# It should be used only for testing purposes.
m = Mnemonic('english')
words = m.to_mnemonic(self.rng.randbytes(32))
self.log.debug('randomized step: generate wallet', words=words)
wallet.unlock(words=words, tx_storage=manager.tx_storage)
return manager
class RunNode:
def create_parser(self) -> ArgumentParser:
from hathor.cli.util import create_parser
parser = create_parser()
parser.add_argument('--hostname',
help='Hostname used to be accessed by other peers')
parser.add_argument('--auto-hostname',
action='store_true',
help='Try to discover the hostname automatically')
parser.add_argument('--testnet',
action='store_true',
help='Connect to Hathor testnet')
parser.add_argument('--test-mode-tx-weight',
action='store_true',
help='Reduces tx weight to 1 for testing purposes')
parser.add_argument('--dns', action='append', help='Seed DNS')
parser.add_argument('--peer', help='json file with peer info')
parser.add_argument(
'--listen',
action='append',
default=[],
help='Address to listen for new connections (eg: tcp:8000)')
parser.add_argument(
'--bootstrap',
action='append',
help='Address to connect to (eg: tcp:127.0.0.1:8000')
parser.add_argument('--status',
type=int,
help='Port to run status server')
parser.add_argument('--stratum',
type=int,
help='Port to run stratum server')
parser.add_argument('--data', help='Data directory')
parser.add_argument('--rocksdb-storage',
action='store_true',
help='Use RocksDB storage backend')
parser.add_argument(
'--wallet',
help=
'Set wallet type. Options are hd (Hierarchical Deterministic) or keypair',
default=None)
parser.add_argument(
'--wallet-enable-api',
action='store_true',
help='Enable wallet API. Must be used with --wallet.'),
parser.add_argument(
'--words', help='Words used to generate the seed for HD Wallet')
parser.add_argument(
'--passphrase',
action='store_true',
help='Passphrase used to generate the seed for HD Wallet')
parser.add_argument('--unlock-wallet',
action='store_true',
help='Ask for password to unlock wallet')
parser.add_argument(
'--wallet-index',
action='store_true',
help=
'Create an index of transactions by address and allow searching queries'
)
parser.add_argument('--prometheus',
action='store_true',
help='Send metric data to Prometheus')
parser.add_argument('--cache',
action='store_true',
help='Use cache for tx storage')
parser.add_argument('--cache-size',
type=int,
help='Number of txs to keep on cache')
parser.add_argument('--cache-interval',
type=int,
help='Cache flush interval')
parser.add_argument('--recursion-limit',
type=int,
help='Set python recursion limit')
parser.add_argument('--allow-mining-without-peers',
action='store_true',
help='Allow mining without peers')
parser.add_argument('--min-block-weight',
type=int,
help='Minimum weight for blocks')
return parser
def prepare(self, args: Namespace) -> None:
import hathor
from hathor.conf import HathorSettings
from hathor.manager import HathorManager, TestMode
from hathor.p2p.peer_discovery import BootstrapPeerDiscovery, DNSPeerDiscovery
from hathor.p2p.peer_id import PeerId
from hathor.p2p.utils import discover_hostname
from hathor.transaction import genesis
from hathor.transaction.storage import (
TransactionStorage,
TransactionCacheStorage,
TransactionCompactStorage,
TransactionMemoryStorage,
)
from hathor.wallet import HDWallet, Wallet
settings = HathorSettings()
if args.recursion_limit:
sys.setrecursionlimit(args.recursion_limit)
if not args.peer:
peer_id = PeerId()
else:
data = json.load(open(args.peer, 'r'))
peer_id = PeerId.create_from_json(data)
print('Hathor v{} (genesis {})'.format(hathor.__version__,
genesis.GENESIS_HASH.hex()[:7]))
print('My peer id is', peer_id.id)
def create_wallet():
if args.wallet == 'hd':
print('Using HDWallet')
kwargs = {
'words': args.words,
}
if args.passphrase:
wallet_passphrase = getpass.getpass(
prompt='HD Wallet passphrase:')
kwargs['passphrase'] = wallet_passphrase.encode()
if args.data:
kwargs['directory'] = args.data
return HDWallet(**kwargs)
elif args.wallet == 'keypair':
print('Using KeyPairWallet')
if args.data:
wallet = Wallet(directory=args.data)
else:
wallet = Wallet()
wallet.flush_to_disk_interval = 5 # seconds
if args.unlock_wallet:
wallet_passwd = getpass.getpass(prompt='Wallet password:'******'Invalid type for wallet')
tx_storage: TransactionStorage
if args.data:
wallet_dir = args.data
print('Using Wallet at {}'.format(wallet_dir))
if args.rocksdb_storage:
from hathor.transaction.storage import TransactionRocksDBStorage
tx_dir = os.path.join(args.data, 'tx.db')
tx_storage = TransactionRocksDBStorage(
path=tx_dir, with_index=(not args.cache))
print('Using TransactionRocksDBStorage at {}'.format(tx_dir))
else:
tx_dir = os.path.join(args.data, 'tx')
tx_storage = TransactionCompactStorage(
path=tx_dir, with_index=(not args.cache))
print('Using TransactionCompactStorage at {}'.format(tx_dir))
if args.cache:
tx_storage = TransactionCacheStorage(tx_storage, reactor)
if args.cache_size:
tx_storage.capacity = args.cache_size
if args.cache_interval:
tx_storage.interval = args.cache_interval
print(
'Using TransactionCacheStorage, capacity {}, interval {}s'.
format(tx_storage.capacity, tx_storage.interval))
tx_storage.start()
else:
# if using MemoryStorage, no need to have cache
tx_storage = TransactionMemoryStorage()
print('Using TransactionMemoryStorage')
self.tx_storage = tx_storage
if args.wallet:
self.wallet = create_wallet()
else:
self.wallet = None
if args.hostname and args.auto_hostname:
print('You cannot use --hostname and --auto-hostname together.')
sys.exit(-1)
if not args.auto_hostname:
hostname = args.hostname
else:
print('Trying to discover your hostname...')
hostname = discover_hostname()
if not hostname:
print('Aborting because we could not discover your hostname.')
print('Try again or run without --auto-hostname.')
sys.exit(-1)
print('Hostname discovered and set to {}'.format(hostname))
network = settings.NETWORK_NAME
self.manager = HathorManager(reactor,
peer_id=peer_id,
network=network,
hostname=hostname,
tx_storage=self.tx_storage,
wallet=self.wallet,
wallet_index=args.wallet_index,
stratum_port=args.stratum,
min_block_weight=args.min_block_weight,
ssl=True)
if args.allow_mining_without_peers:
self.manager.allow_mining_without_peers()
dns_hosts = []
if settings.BOOTSTRAP_DNS:
dns_hosts.extend(settings.BOOTSTRAP_DNS)
if args.dns:
dns_hosts.extend(args.dns)
if dns_hosts:
self.manager.add_peer_discovery(DNSPeerDiscovery(dns_hosts))
if args.bootstrap:
self.manager.add_peer_discovery(
BootstrapPeerDiscovery(args.bootstrap))
if args.test_mode_tx_weight:
self.manager.test_mode = TestMode.TEST_TX_WEIGHT
if self.wallet:
self.wallet.test_mode = True
for description in args.listen:
self.manager.add_listen_address(description)
self.start_manager()
self.register_resources(args)
def start_manager(self) -> None:
self.manager.start()
def register_resources(self, args: Namespace) -> None:
from hathor.conf import HathorSettings
from hathor.p2p.resources import AddPeersResource, MiningInfoResource, MiningResource, StatusResource
from hathor.prometheus import PrometheusMetricsExporter
from hathor.resources import ProfilerResource
from hathor.transaction.resources import (
DashboardTransactionResource,
DecodeTxResource,
GraphvizLegacyResource,
GraphvizFullResource,
GraphvizNeighboursResource,
PushTxResource,
TipsHistogramResource,
TipsResource,
TransactionAccWeightResource,
TransactionResource,
)
from hathor.version_resource import VersionResource
from hathor.wallet.resources import (
AddressResource,
BalanceResource,
HistoryResource,
LockWalletResource,
SendTokensResource,
SignTxResource,
StateWalletResource,
UnlockWalletResource,
)
from hathor.wallet.resources.thin_wallet import (
AddressHistoryResource, SendTokensResource as
SendTokensThinResource, TokenHistoryResource, TokenResource)
from hathor.wallet.resources.nano_contracts import (
NanoContractDecodeResource,
NanoContractExecuteResource,
NanoContractMatchValueResource,
)
from hathor.websocket import HathorAdminWebsocketFactory, WebsocketStatsResource
from hathor.stratum.resources import MiningStatsResource
settings = HathorSettings()
if args.prometheus:
kwargs: Dict[str, Any] = {'metrics': self.manager.metrics}
if args.data:
kwargs['path'] = os.path.join(args.data, 'prometheus')
else:
raise ValueError(
'To run prometheus exporter you must have a data path')
prometheus = PrometheusMetricsExporter(**kwargs)
prometheus.start()
if args.status:
# TODO get this from a file. How should we do with the factory?
root = Resource()
wallet_resource = Resource()
root.putChild(b'wallet', wallet_resource)
thin_wallet_resource = Resource()
root.putChild(b'thin_wallet', thin_wallet_resource)
contracts_resource = Resource()
wallet_resource.putChild(b'nano-contract', contracts_resource)
p2p_resource = Resource()
root.putChild(b'p2p', p2p_resource)
graphviz = GraphvizLegacyResource(self.manager)
# XXX: reach the resource through /graphviz/ too, previously it was a leaf so this wasn't a problem
graphviz.putChild(b'', graphviz)
for fmt in ['dot', 'pdf', 'png', 'jpg']:
bfmt = fmt.encode('ascii')
graphviz.putChild(
b'full.' + bfmt,
GraphvizFullResource(self.manager, format=fmt))
graphviz.putChild(
b'neighbours.' + bfmt,
GraphvizNeighboursResource(self.manager, format=fmt))
resources = (
(b'status', StatusResource(self.manager), root),
(b'version', VersionResource(self.manager), root),
(b'mining', MiningResource(self.manager), root),
(b'getmininginfo', MiningInfoResource(self.manager), root),
(b'decode_tx', DecodeTxResource(self.manager), root),
(b'push_tx', PushTxResource(self.manager), root),
(b'graphviz', graphviz, root),
(b'tips-histogram', TipsHistogramResource(self.manager), root),
(b'tips', TipsResource(self.manager), root),
(b'transaction', TransactionResource(self.manager), root),
(b'transaction_acc_weight',
TransactionAccWeightResource(self.manager), root),
(b'dashboard_tx', DashboardTransactionResource(self.manager),
root),
(b'profiler', ProfilerResource(self.manager), root),
# /thin_wallet
(b'address_history', AddressHistoryResource(self.manager),
thin_wallet_resource),
(b'send_tokens', SendTokensThinResource(self.manager),
thin_wallet_resource),
(b'token', TokenResource(self.manager), thin_wallet_resource),
(b'token_history', TokenHistoryResource(self.manager),
thin_wallet_resource),
# /wallet/nano-contract
(b'match-value', NanoContractMatchValueResource(self.manager),
contracts_resource),
(b'decode', NanoContractDecodeResource(self.manager),
contracts_resource),
(b'execute', NanoContractExecuteResource(self.manager),
contracts_resource),
# /p2p
(b'peers', AddPeersResource(self.manager), p2p_resource),
)
for url_path, resource, parent in resources:
parent.putChild(url_path, resource)
if self.manager.stratum_factory is not None:
root.putChild(b'miners', MiningStatsResource(self.manager))
if self.wallet and args.wallet_enable_api:
wallet_resources = (
# /wallet
(b'balance', BalanceResource(self.manager), wallet_resource
),
(b'history', HistoryResource(self.manager),
wallet_resource),
(b'address', AddressResource(self.manager),
wallet_resource),
(b'send_tokens', SendTokensResource(self.manager),
wallet_resource),
(b'sign_tx', SignTxResource(self.manager),
wallet_resource),
(b'unlock', UnlockWalletResource(self.manager),
wallet_resource),
(b'lock', LockWalletResource(self.manager),
wallet_resource),
(b'state', StateWalletResource(self.manager),
wallet_resource),
)
for url_path, resource, parent in wallet_resources:
parent.putChild(url_path, resource)
# Websocket resource
ws_factory = HathorAdminWebsocketFactory(
metrics=self.manager.metrics,
wallet_index=self.manager.tx_storage.wallet_index)
ws_factory.start()
resource = WebSocketResource(ws_factory)
root.putChild(b"ws", resource)
ws_factory.subscribe(self.manager.pubsub)
# Websocket stats resource
root.putChild(b'websocket_stats',
WebsocketStatsResource(ws_factory))
real_root = Resource()
real_root.putChild(settings.API_VERSION_PREFIX.encode('ascii'),
root)
status_server = server.Site(real_root)
reactor.listenTCP(args.status, status_server)
# Set websocket factory in metrics
self.manager.metrics.websocket_factory = ws_factory
def __init__(self, *, argv=None):
if argv is None:
import sys
argv = sys.argv[1:]
self.parser = self.create_parser()
args = self.parse_args(argv)
if args.testnet:
if not os.environ.get('HATHOR_CONFIG_FILE'):
os.environ['HATHOR_CONFIG_FILE'] = 'hathor.conf.testnet'
self.prepare(args)
def parse_args(self, argv: List[str]) -> Namespace:
return self.parser.parse_args(argv)
def run(self) -> None:
reactor.run()
def prepare(self, args: Namespace) -> None:
import hathor
from hathor.conf import HathorSettings
from hathor.manager import HathorManager, TestMode
from hathor.p2p.peer_discovery import BootstrapPeerDiscovery, DNSPeerDiscovery
from hathor.p2p.peer_id import PeerId
from hathor.p2p.utils import discover_hostname
from hathor.transaction import genesis
from hathor.transaction.storage import (
TransactionStorage,
TransactionCacheStorage,
TransactionCompactStorage,
TransactionMemoryStorage,
)
from hathor.wallet import HDWallet, Wallet
settings = HathorSettings()
if args.recursion_limit:
sys.setrecursionlimit(args.recursion_limit)
if not args.peer:
peer_id = PeerId()
else:
data = json.load(open(args.peer, 'r'))
peer_id = PeerId.create_from_json(data)
print('Hathor v{} (genesis {})'.format(hathor.__version__,
genesis.GENESIS_HASH.hex()[:7]))
print('My peer id is', peer_id.id)
def create_wallet():
if args.wallet == 'hd':
print('Using HDWallet')
kwargs = {
'words': args.words,
}
if args.passphrase:
wallet_passphrase = getpass.getpass(
prompt='HD Wallet passphrase:')
kwargs['passphrase'] = wallet_passphrase.encode()
if args.data:
kwargs['directory'] = args.data
return HDWallet(**kwargs)
elif args.wallet == 'keypair':
print('Using KeyPairWallet')
if args.data:
wallet = Wallet(directory=args.data)
else:
wallet = Wallet()
wallet.flush_to_disk_interval = 5 # seconds
if args.unlock_wallet:
wallet_passwd = getpass.getpass(prompt='Wallet password:'******'Invalid type for wallet')
tx_storage: TransactionStorage
if args.data:
wallet_dir = args.data
print('Using Wallet at {}'.format(wallet_dir))
if args.rocksdb_storage:
from hathor.transaction.storage import TransactionRocksDBStorage
tx_dir = os.path.join(args.data, 'tx.db')
tx_storage = TransactionRocksDBStorage(
path=tx_dir, with_index=(not args.cache))
print('Using TransactionRocksDBStorage at {}'.format(tx_dir))
else:
tx_dir = os.path.join(args.data, 'tx')
tx_storage = TransactionCompactStorage(
path=tx_dir, with_index=(not args.cache))
print('Using TransactionCompactStorage at {}'.format(tx_dir))
if args.cache:
tx_storage = TransactionCacheStorage(tx_storage, reactor)
if args.cache_size:
tx_storage.capacity = args.cache_size
if args.cache_interval:
tx_storage.interval = args.cache_interval
print(
'Using TransactionCacheStorage, capacity {}, interval {}s'.
format(tx_storage.capacity, tx_storage.interval))
tx_storage.start()
else:
# if using MemoryStorage, no need to have cache
tx_storage = TransactionMemoryStorage()
print('Using TransactionMemoryStorage')
self.tx_storage = tx_storage
if args.wallet:
self.wallet = create_wallet()
else:
self.wallet = None
if args.hostname and args.auto_hostname:
print('You cannot use --hostname and --auto-hostname together.')
sys.exit(-1)
if not args.auto_hostname:
hostname = args.hostname
else:
print('Trying to discover your hostname...')
hostname = discover_hostname()
if not hostname:
print('Aborting because we could not discover your hostname.')
print('Try again or run without --auto-hostname.')
sys.exit(-1)
print('Hostname discovered and set to {}'.format(hostname))
network = settings.NETWORK_NAME
self.manager = HathorManager(reactor,
peer_id=peer_id,
network=network,
hostname=hostname,
tx_storage=self.tx_storage,
wallet=self.wallet,
wallet_index=args.wallet_index,
stratum_port=args.stratum,
min_block_weight=args.min_block_weight,
ssl=True)
if args.allow_mining_without_peers:
self.manager.allow_mining_without_peers()
dns_hosts = []
if settings.BOOTSTRAP_DNS:
dns_hosts.extend(settings.BOOTSTRAP_DNS)
if args.dns:
dns_hosts.extend(args.dns)
if dns_hosts:
self.manager.add_peer_discovery(DNSPeerDiscovery(dns_hosts))
if args.bootstrap:
self.manager.add_peer_discovery(
BootstrapPeerDiscovery(args.bootstrap))
if args.test_mode_tx_weight:
self.manager.test_mode = TestMode.TEST_TX_WEIGHT
if self.wallet:
self.wallet.test_mode = True
for description in args.listen:
self.manager.add_listen_address(description)
self.start_manager()
self.register_resources(args)
def create_peer(self,
network,
peer_id=None,
wallet=None,
tx_storage=None,
unlock_wallet=True,
wallet_index=False,
capabilities=None,
full_verification=True,
enable_sync_v1=None,
enable_sync_v2=None,
checkpoints=None):
if enable_sync_v1 is None:
assert hasattr(self, '_enable_sync_v1'), (
'`_enable_sync_v1` has no default by design, either set one on '
'the test class or pass `enable_sync_v1` by argument')
enable_sync_v1 = self._enable_sync_v1
if enable_sync_v2 is None:
assert hasattr(self, '_enable_sync_v2'), (
'`_enable_sync_v2` has no default by design, either set one on '
'the test class or pass `enable_sync_v2` by argument')
enable_sync_v2 = self._enable_sync_v2
assert enable_sync_v1 or enable_sync_v2, 'enable at least one sync version'
if peer_id is None:
peer_id = PeerId()
if not wallet:
wallet = self._create_test_wallet()
if unlock_wallet:
wallet.unlock(b'MYPASS')
if tx_storage is None:
if self.use_memory_storage:
from hathor.transaction.storage.memory_storage import TransactionMemoryStorage
tx_storage = TransactionMemoryStorage()
else:
from hathor.transaction.storage.rocksdb_storage import TransactionRocksDBStorage
directory = tempfile.mkdtemp()
self.tmpdirs.append(directory)
tx_storage = TransactionRocksDBStorage(directory)
manager = HathorManager(
self.clock,
peer_id=peer_id,
network=network,
wallet=wallet,
tx_storage=tx_storage,
wallet_index=wallet_index,
capabilities=capabilities,
rng=self.rng,
enable_sync_v1=enable_sync_v1,
enable_sync_v2=enable_sync_v2,
checkpoints=checkpoints,
)
# XXX: just making sure that tests set this up correctly
if enable_sync_v2:
assert SyncVersion.V2 in manager.connections._sync_factories
else:
assert SyncVersion.V2 not in manager.connections._sync_factories
if enable_sync_v1:
assert SyncVersion.V1 in manager.connections._sync_factories
else:
assert SyncVersion.V1 not in manager.connections._sync_factories
manager.avg_time_between_blocks = 0.0001
manager._full_verification = full_verification
manager.start()
self.run_to_completion()
return manager
def add_custom_tx(manager: HathorManager, tx_inputs: List[Tuple[Transaction, int]], *, n_outputs: int = 1,
base_parent: Optional[Transaction] = None, weight: Optional[float] = None) -> Transaction:
"""Add a custom tx based on the gen_custom_tx(...) method."""
tx = gen_custom_tx(manager, tx_inputs, n_outputs=n_outputs, base_parent=base_parent, weight=weight)
manager.propagate_tx(tx, fails_silently=False)
return tx
def add_new_double_spending(manager: HathorManager, *, use_same_parents: bool = False,
tx: Optional[Transaction] = None, weight: float = 1) -> Transaction:
tx = gen_new_double_spending(manager, use_same_parents=use_same_parents, tx=tx, weight=weight)
manager.propagate_tx(tx, fails_silently=False)
return tx
class _TransactionStorageTest(unittest.TestCase):
def setUp(self, tx_storage, reactor=None):
if not reactor:
self.reactor = Clock()
else:
self.reactor = reactor
self.reactor.advance(time.time())
self.tx_storage = tx_storage
tx_storage._manually_initialize()
self.genesis = self.tx_storage.get_all_genesis()
self.genesis_blocks = [tx for tx in self.genesis if tx.is_block]
self.genesis_txs = [tx for tx in self.genesis if not tx.is_block]
from hathor.manager import HathorManager
self.tmpdir = tempfile.mkdtemp(dir='/tmp/')
wallet = Wallet(directory=self.tmpdir)
wallet.unlock(b'teste')
self.manager = HathorManager(self.reactor,
tx_storage=self.tx_storage,
wallet=wallet)
self.tx_storage.wallet_index = WalletIndex(self.manager.pubsub)
self.tx_storage.tokens_index = TokensIndex()
block_parents = [
tx.hash for tx in chain(self.genesis_blocks, self.genesis_txs)
]
output = TxOutput(
200, bytes.fromhex('1e393a5ce2ff1c98d4ff6892f2175100f2dad049'))
self.block = Block(timestamp=MIN_TIMESTAMP,
weight=12,
outputs=[output],
parents=block_parents,
nonce=100781,
storage=tx_storage)
self.block.resolve()
self.block.verify()
tx_parents = [tx.hash for tx in self.genesis_txs]
tx_input = TxInput(
tx_id=self.genesis_blocks[0].hash,
index=0,
data=bytes.fromhex(
'46304402203470cb9818c9eb842b0c433b7e2b8aded0a51f5903e971649e870763d0266a'
'd2022049b48e09e718c4b66a0f3178ef92e4d60ee333d2d0e25af8868acf5acbb35aaa583'
'056301006072a8648ce3d020106052b8104000a034200042ce7b94cba00b654d4308f8840'
'7345cacb1f1032fb5ac80407b74d56ed82fb36467cb7048f79b90b1cf721de57e942c5748'
'620e78362cf2d908e9057ac235a63'))
self.tx = Transaction(
timestamp=MIN_TIMESTAMP + 2,
weight=10,
nonce=932049,
inputs=[tx_input],
outputs=[output],
tokens=[
bytes.fromhex(
'0023be91834c973d6a6ddd1a0ae411807b7c8ef2a015afb5177ee64b666ce602'
)
],
parents=tx_parents,
storage=tx_storage)
self.tx.resolve()
# Disable weakref to test the internal methods. Otherwise, most methods return objects from weakref.
self.tx_storage._disable_weakref()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_genesis(self):
self.assertEqual(1, len(self.genesis_blocks))
self.assertEqual(2, len(self.genesis_txs))
for tx in self.genesis:
tx.verify()
for tx in self.genesis:
tx2 = self.tx_storage.get_transaction(tx.hash)
self.assertEqual(tx, tx2)
self.assertTrue(self.tx_storage.transaction_exists(tx.hash))
def test_storage_basic(self):
self.assertEqual(1, self.tx_storage.get_block_count())
self.assertEqual(2, self.tx_storage.get_tx_count())
self.assertEqual(3, self.tx_storage.get_count_tx_blocks())
block_parents_hash = [
x.data for x in self.tx_storage.get_block_tips()
]
self.assertEqual(1, len(block_parents_hash))
self.assertEqual(block_parents_hash, [self.genesis_blocks[0].hash])
tx_parents_hash = [x.data for x in self.tx_storage.get_tx_tips()]
self.assertEqual(2, len(tx_parents_hash))
self.assertEqual(
set(tx_parents_hash),
{self.genesis_txs[0].hash, self.genesis_txs[1].hash})
def validate_save(self, obj):
self.tx_storage.save_transaction(obj)
loaded_obj1 = self.tx_storage.get_transaction(obj.hash)
self.assertTrue(self.tx_storage.transaction_exists(obj.hash))
self.assertEqual(obj, loaded_obj1)
self.assertEqual(len(obj.get_funds_struct()),
len(loaded_obj1.get_funds_struct()))
self.assertEqual(bytes(obj), bytes(loaded_obj1))
self.assertEqual(obj.to_json(), loaded_obj1.to_json())
self.assertEqual(obj.is_block, loaded_obj1.is_block)
# Testing add and remove from cache
if self.tx_storage.with_index:
if obj.is_block:
self.assertTrue(obj.hash in self.tx_storage.block_index.
tips_index.tx_last_interval)
else:
self.assertTrue(obj.hash in self.tx_storage.tx_index.
tips_index.tx_last_interval)
self.tx_storage._del_from_cache(obj)
if self.tx_storage.with_index:
if obj.is_block:
self.assertFalse(obj.hash in self.tx_storage.block_index.
tips_index.tx_last_interval)
else:
self.assertFalse(obj.hash in self.tx_storage.tx_index.
tips_index.tx_last_interval)
self.tx_storage._add_to_cache(obj)
if self.tx_storage.with_index:
if obj.is_block:
self.assertTrue(obj.hash in self.tx_storage.block_index.
tips_index.tx_last_interval)
else:
self.assertTrue(obj.hash in self.tx_storage.tx_index.
tips_index.tx_last_interval)
def test_save_block(self):
self.validate_save(self.block)
def test_save_tx(self):
self.validate_save(self.tx)
def test_save_token_creation_tx(self):
tx = create_tokens(self.manager, propagate=False)
self.validate_save(tx)
def _validate_not_in_index(self, tx, index):
tips = index.tips_index[self.tx.timestamp]
self.assertNotIn(self.tx.hash, [x.data for x in tips])
self.assertNotIn(self.tx.hash, index.tips_index.tx_last_interval)
self.assertIsNone(index.txs_index.find_tx_index(tx))
def _test_remove_tx_or_block(self, tx):
self.validate_save(tx)
self.tx_storage.remove_transaction(tx)
with self.assertRaises(TransactionDoesNotExist):
self.tx_storage.get_transaction(tx.hash)
if hasattr(self.tx_storage, 'all_index'):
self._validate_not_in_index(tx, self.tx_storage.all_index)
if tx.is_block:
if hasattr(self.tx_storage, 'block_index'):
self._validate_not_in_index(tx,
self.tx_storage.block_index)
else:
if hasattr(self.tx_storage, 'tx_index'):
self._validate_not_in_index(tx, self.tx_storage.tx_index)
# Check wallet index.
wallet_index = self.tx_storage.wallet_index
addresses = wallet_index._get_addresses(tx)
for address in addresses:
self.assertNotIn(tx.hash, wallet_index.index[address])
# TODO Check self.tx_storage.tokens_index
# Try to remove twice. It is supposed to do nothing.
self.tx_storage.remove_transaction(tx)
def test_remove_tx(self):
self._test_remove_tx_or_block(self.tx)
def test_remove_block(self):
self._test_remove_tx_or_block(self.block)
def test_shared_memory(self):
# Enable weakref to this test only.
self.tx_storage._enable_weakref()
self.validate_save(self.block)
self.validate_save(self.tx)
for tx in [self.tx, self.block]:
# just making sure, if it is genesis the test is wrong
self.assertFalse(tx.is_genesis)
# load transactions twice
tx1 = self.tx_storage.get_transaction(tx.hash)
tx2 = self.tx_storage.get_transaction(tx.hash)
# naturally they should be equal, but this time so do the objects
self.assertTrue(tx1 == tx2)
self.assertTrue(tx1 is tx2)
meta1 = tx1.get_metadata()
meta2 = tx2.get_metadata()
# and naturally the metadata too
self.assertTrue(meta1 == meta2)
self.assertTrue(meta1 is meta2)
def test_get_wrong_tx(self):
hex_error = bytes.fromhex(
'00001c5c0b69d13b05534c94a69b2c8272294e6b0c536660a3ac264820677024'
)
with self.assertRaises(TransactionDoesNotExist):
self.tx_storage.get_transaction(hex_error)
def test_save_metadata(self):
# Saving genesis metadata
self.tx_storage.save_transaction(self.genesis_txs[0],
only_metadata=True)
tx = self.block
# First we save to the storage
self.tx_storage.save_transaction(tx)
metadata = tx.get_metadata()
metadata.spent_outputs[1].append(self.genesis_blocks[0].hash)
random_tx = bytes.fromhex(
'0000222e64683b966b4268f387c269915cc61f6af5329823a93e3696cb0f2222'
)
metadata.children.append(random_tx)
self.tx_storage.save_transaction(tx, only_metadata=True)
tx2 = self.tx_storage.get_transaction(tx.hash)
metadata2 = tx2.get_metadata()
self.assertEqual(metadata, metadata2)
total = 0
for tx in self.tx_storage.get_all_transactions():
total += 1
self.assertEqual(total, 4)
def test_storage_new_blocks(self):
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(tip_blocks, [self.genesis_blocks[0].hash])
block1 = self._add_new_block()
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(tip_blocks, [block1.hash])
block2 = self._add_new_block()
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(tip_blocks, [block2.hash])
# Block3 has the same parents as block2.
block3 = self._add_new_block(parents=block2.parents)
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(set(tip_blocks), {block2.hash, block3.hash})
# Re-generate caches to test topological sort.
self.tx_storage._manually_initialize()
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(set(tip_blocks), {block2.hash, block3.hash})
def test_token_list(self):
tx = self.tx
self.validate_save(tx)
# 2 token uids
tx.tokens.append(
bytes.fromhex(
'00001c5c0b69d13b05534c94a69b2c8272294e6b0c536660a3ac264820677024'
))
tx.resolve()
self.validate_save(tx)
# no tokens
tx.tokens = []
tx.resolve()
self.validate_save(tx)
def _add_new_block(self, parents=None):
block = self.manager.generate_mining_block()
block.data = b'Testing, testing, 1, 2, 3... testing, testing...'
if parents is not None:
block.parents = parents
block.weight = 10
self.assertTrue(block.resolve())
block.verify()
self.manager.tx_storage.save_transaction(block)
self.reactor.advance(5)
return block
def test_topological_sort(self):
self.manager.test_mode = TestMode.TEST_ALL_WEIGHT
_total = 0
blocks = add_new_blocks(self.manager, 1, advance_clock=1)
_total += len(blocks)
blocks = add_blocks_unlock_reward(self.manager)
_total += len(blocks)
add_new_transactions(self.manager, 1, advance_clock=1)[0]
total = 0
for tx in self.tx_storage._topological_sort():
total += 1
# added blocks + genesis txs + added tx
self.assertEqual(total, _total + 3 + 1)
def test_get_best_block_weight(self):
block = self._add_new_block()
weight = self.tx_storage.get_weight_best_block()
self.assertEqual(block.weight, weight)
def test_double_start(self):
manager = HathorManager(self.clock, tx_storage=self.tx_storage)
manager.start()
with self.assertRaises(Exception):
manager.start()
def add_new_double_spending(manager: HathorManager, *, use_same_parents: bool = False) -> Transaction:
tx = gen_new_double_spending(manager, use_same_parents=use_same_parents)
manager.propagate_tx(tx, fails_silently=False)
return tx
class RunNode:
UNSAFE_ARGUMENTS: List[Tuple[str, Callable[[Namespace], bool]]] = [
('--test-mode-tx-weight', lambda args: bool(args.test_mode_tx_weight)),
('--enable-crash-api', lambda args: bool(args.enable_crash_api)),
('--x-sync-bridge', lambda args: bool(args.x_sync_bridge)),
('--x-sync-v2-only', lambda args: bool(args.x_sync_v2_only)),
]
def create_parser(self) -> ArgumentParser:
from hathor.cli.util import create_parser
parser = create_parser()
parser.add_argument('--hostname', help='Hostname used to be accessed by other peers')
parser.add_argument('--auto-hostname', action='store_true', help='Try to discover the hostname automatically')
parser.add_argument('--unsafe-mode',
help='Enable unsafe parameters. **NEVER USE IT IN PRODUCTION ENVIRONMENT**')
parser.add_argument('--testnet', action='store_true', help='Connect to Hathor testnet')
parser.add_argument('--test-mode-tx-weight', action='store_true',
help='Reduces tx weight to 1 for testing purposes')
parser.add_argument('--dns', action='append', help='Seed DNS')
parser.add_argument('--peer', help='json file with peer info')
parser.add_argument('--listen', action='append', default=[],
help='Address to listen for new connections (eg: tcp:8000)')
parser.add_argument('--bootstrap', action='append', help='Address to connect to (eg: tcp:127.0.0.1:8000')
parser.add_argument('--status', type=int, help='Port to run status server')
parser.add_argument('--stratum', type=int, help='Port to run stratum server')
parser.add_argument('--data', help='Data directory')
storage = parser.add_mutually_exclusive_group()
storage.add_argument('--rocksdb-storage', action='store_true', help='Use RocksDB storage backend (default)')
storage.add_argument('--memory-storage', action='store_true', help='Do not use any storage')
storage.add_argument('--json-storage', action='store_true', help='Use legacy JSON storage (not recommended)')
parser.add_argument('--rocksdb-cache', type=int, help='RocksDB block-table cache size (bytes)', default=None)
parser.add_argument('--wallet', help='Set wallet type. Options are hd (Hierarchical Deterministic) or keypair',
default=None)
parser.add_argument('--wallet-enable-api', action='store_true',
help='Enable wallet API. Must be used with --wallet.'),
parser.add_argument('--words', help='Words used to generate the seed for HD Wallet')
parser.add_argument('--passphrase', action='store_true',
help='Passphrase used to generate the seed for HD Wallet')
parser.add_argument('--unlock-wallet', action='store_true', help='Ask for password to unlock wallet')
parser.add_argument('--wallet-index', action='store_true',
help='Create an index of transactions by address and allow searching queries')
parser.add_argument('--prometheus', action='store_true', help='Send metric data to Prometheus')
parser.add_argument('--cache', action='store_true', help='Use cache for tx storage')
parser.add_argument('--cache-size', type=int, help='Number of txs to keep on cache')
parser.add_argument('--cache-interval', type=int, help='Cache flush interval')
parser.add_argument('--recursion-limit', type=int, help='Set python recursion limit')
parser.add_argument('--allow-mining-without-peers', action='store_true', help='Allow mining without peers')
fvargs = parser.add_mutually_exclusive_group()
fvargs.add_argument('--x-full-verification', action='store_true', help='Fully validate the local database')
fvargs.add_argument('--x-fast-init-beta', action='store_true',
help='Execute a fast initialization, which skips some transaction verifications. '
'This is still a beta feature as it may cause issues when restarting the full node '
'after a crash.')
parser.add_argument('--procname-prefix', help='Add a prefix to the process name', default='')
parser.add_argument('--allow-non-standard-script', action='store_true', help='Accept non-standard scripts on '
'/push-tx API')
parser.add_argument('--max-output-script-size', type=int, default=None, help='Custom max accepted script size '
'on /push-tx API')
parser.add_argument('--sentry-dsn', help='Sentry DSN')
parser.add_argument('--enable-debug-api', action='store_true', help='Enable _debug/* endpoints')
parser.add_argument('--enable-crash-api', action='store_true', help='Enable _crash/* endpoints')
v2args = parser.add_mutually_exclusive_group()
v2args.add_argument('--x-sync-bridge', action='store_true',
help='Enable support for running both sync protocols. DO NOT ENABLE, IT WILL BREAK.')
v2args.add_argument('--x-sync-v2-only', action='store_true',
help='Disable support for running sync-v1. DO NOT ENABLE, IT WILL BREAK.')
parser.add_argument('--x-localhost-only', action='store_true', help='Only connect to peers on localhost')
parser.add_argument('--x-rocksdb-indexes', action='store_true', help='Use RocksDB indexes (currently opt-in)')
return parser
def prepare(self, args: Namespace) -> None:
import hathor
from hathor.cli.util import check_or_exit
from hathor.conf import HathorSettings
from hathor.conf.get_settings import get_settings_module
from hathor.daa import TestMode, _set_test_mode
from hathor.manager import HathorManager
from hathor.p2p.peer_discovery import BootstrapPeerDiscovery, DNSPeerDiscovery
from hathor.p2p.peer_id import PeerId
from hathor.p2p.utils import discover_hostname
from hathor.transaction import genesis
from hathor.transaction.storage import (
TransactionCacheStorage,
TransactionCompactStorage,
TransactionMemoryStorage,
TransactionRocksDBStorage,
TransactionStorage,
)
from hathor.wallet import HDWallet, Wallet
settings = HathorSettings()
settings_module = get_settings_module() # only used for logging its location
self.log = logger.new()
from setproctitle import setproctitle
setproctitle('{}hathor-core'.format(args.procname_prefix))
if args.recursion_limit:
sys.setrecursionlimit(args.recursion_limit)
else:
sys.setrecursionlimit(5000)
try:
import resource
except ModuleNotFoundError:
pass
else:
(nofile_soft, _) = resource.getrlimit(resource.RLIMIT_NOFILE)
if nofile_soft < 256:
print('Maximum number of open file descriptors is too low. Minimum required is 256.')
sys.exit(-2)
if not args.peer:
peer_id = PeerId()
else:
data = json.load(open(args.peer, 'r'))
peer_id = PeerId.create_from_json(data)
python = f'{platform.python_version()}-{platform.python_implementation()}'
self.check_unsafe_arguments(args)
self.log.info(
'hathor-core v{hathor}',
hathor=hathor.__version__,
pid=os.getpid(),
genesis=genesis.GENESIS_HASH.hex()[:7],
my_peer_id=str(peer_id.id),
python=python,
platform=platform.platform(),
settings=settings_module.__file__,
)
def create_wallet():
if args.wallet == 'hd':
kwargs = {
'words': args.words,
}
if args.passphrase:
wallet_passphrase = getpass.getpass(prompt='HD Wallet passphrase:')
kwargs['passphrase'] = wallet_passphrase.encode()
if args.data:
kwargs['directory'] = args.data
return HDWallet(**kwargs)
elif args.wallet == 'keypair':
print('Using KeyPairWallet')
if args.data:
wallet = Wallet(directory=args.data)
else:
wallet = Wallet()
wallet.flush_to_disk_interval = 5 # seconds
if args.unlock_wallet:
wallet_passwd = getpass.getpass(prompt='Wallet password:'******'Invalid type for wallet')
tx_storage: TransactionStorage
if args.memory_storage:
check_or_exit(not args.data, '--data should not be used with --memory-storage')
# if using MemoryStorage, no need to have cache
tx_storage = TransactionMemoryStorage()
assert not args.x_rocksdb_indexes, 'RocksDB indexes require RocksDB data'
self.log.info('with storage', storage_class=type(tx_storage).__name__)
elif args.json_storage:
check_or_exit(args.data, '--data is expected')
assert not args.x_rocksdb_indexes, 'RocksDB indexes require RocksDB data'
tx_storage = TransactionCompactStorage(path=args.data, with_index=(not args.cache))
else:
check_or_exit(args.data, '--data is expected')
if args.rocksdb_storage:
self.log.warn('--rocksdb-storage is now implied, no need to specify it')
cache_capacity = args.rocksdb_cache
use_memory_indexes = not args.x_rocksdb_indexes
tx_storage = TransactionRocksDBStorage(path=args.data, with_index=(not args.cache),
cache_capacity=cache_capacity,
use_memory_indexes=use_memory_indexes)
self.log.info('with storage', storage_class=type(tx_storage).__name__, path=args.data)
if args.cache:
check_or_exit(not args.memory_storage, '--cache should not be used with --memory-storage')
tx_storage = TransactionCacheStorage(tx_storage, reactor)
if args.cache_size:
tx_storage.capacity = args.cache_size
if args.cache_interval:
tx_storage.interval = args.cache_interval
self.log.info('with cache', capacity=tx_storage.capacity, interval=tx_storage.interval)
self.tx_storage = tx_storage
self.log.info('with indexes', indexes_class=type(tx_storage.indexes).__name__)
if args.wallet:
self.wallet = create_wallet()
self.log.info('with wallet', wallet=self.wallet, path=args.data)
else:
self.wallet = None
if args.hostname and args.auto_hostname:
print('You cannot use --hostname and --auto-hostname together.')
sys.exit(-1)
if not args.auto_hostname:
hostname = args.hostname
else:
print('Trying to discover your hostname...')
hostname = discover_hostname()
if not hostname:
print('Aborting because we could not discover your hostname.')
print('Try again or run without --auto-hostname.')
sys.exit(-1)
print('Hostname discovered and set to {}'.format(hostname))
network = settings.NETWORK_NAME
enable_sync_v1 = not args.x_sync_v2_only
enable_sync_v2 = args.x_sync_v2_only or args.x_sync_bridge
self.manager = HathorManager(
reactor,
peer_id=peer_id,
network=network,
hostname=hostname,
tx_storage=self.tx_storage,
wallet=self.wallet,
wallet_index=args.wallet_index,
stratum_port=args.stratum,
ssl=True,
checkpoints=settings.CHECKPOINTS,
enable_sync_v1=enable_sync_v1,
enable_sync_v2=enable_sync_v2,
soft_voided_tx_ids=set(settings.SOFT_VOIDED_TX_IDS),
)
if args.allow_mining_without_peers:
self.manager.allow_mining_without_peers()
if args.x_localhost_only:
self.manager.connections.localhost_only = True
dns_hosts = []
if settings.BOOTSTRAP_DNS:
dns_hosts.extend(settings.BOOTSTRAP_DNS)
if args.dns:
dns_hosts.extend(args.dns)
if dns_hosts:
self.manager.add_peer_discovery(DNSPeerDiscovery(dns_hosts))
if args.bootstrap:
self.manager.add_peer_discovery(BootstrapPeerDiscovery(args.bootstrap))
if args.test_mode_tx_weight:
_set_test_mode(TestMode.TEST_TX_WEIGHT)
if self.wallet:
self.wallet.test_mode = True
if args.x_full_verification:
self.manager._full_verification = True
if args.x_fast_init_beta:
self.log.warn('--x-fast-init-beta is now the default, no need to specify it')
for description in args.listen:
self.manager.add_listen_address(description)
self.start_manager(args)
self.register_resources(args)
def start_sentry_if_possible(self, args: Namespace) -> None:
"""Start Sentry integration if possible."""
if not args.sentry_dsn:
return
self.log.info('Starting Sentry', dsn=args.sentry_dsn)
try:
import sentry_sdk
from structlog_sentry import SentryProcessor # noqa: F401
except ModuleNotFoundError:
self.log.error('Please use `poetry install -E sentry` for enabling Sentry.')
sys.exit(-3)
import hathor
from hathor.conf import HathorSettings
settings = HathorSettings()
sentry_sdk.init(
dsn=args.sentry_dsn,
release=hathor.__version__,
environment=settings.NETWORK_NAME,
)
def start_manager(self, args: Namespace) -> None:
self.start_sentry_if_possible(args)
self.manager.start()
def register_resources(self, args: Namespace) -> None:
from hathor.conf import HathorSettings
from hathor.debug_resources import (
DebugCrashResource,
DebugLogResource,
DebugMessAroundResource,
DebugPrintResource,
DebugRaiseResource,
DebugRejectResource,
)
from hathor.mining.ws import MiningWebsocketFactory
from hathor.p2p.resources import AddPeersResource, MiningInfoResource, MiningResource, StatusResource
from hathor.profiler import get_cpu_profiler
from hathor.profiler.resources import CPUProfilerResource, ProfilerResource
from hathor.prometheus import PrometheusMetricsExporter
from hathor.transaction.resources import (
BlockAtHeightResource,
CreateTxResource,
DashboardTransactionResource,
DecodeTxResource,
GetBlockTemplateResource,
GraphvizFullResource,
GraphvizNeighboursResource,
MempoolResource,
PushTxResource,
SubmitBlockResource,
TransactionAccWeightResource,
TransactionResource,
TxParentsResource,
ValidateAddressResource,
)
from hathor.version_resource import VersionResource
from hathor.wallet.resources import (
AddressResource,
BalanceResource,
HistoryResource,
LockWalletResource,
SendTokensResource,
SignTxResource,
StateWalletResource,
UnlockWalletResource,
)
from hathor.wallet.resources.nano_contracts import (
NanoContractDecodeResource,
NanoContractExecuteResource,
NanoContractMatchValueResource,
)
from hathor.wallet.resources.thin_wallet import (
AddressBalanceResource,
AddressHistoryResource,
AddressSearchResource,
SendTokensResource as SendTokensThinResource,
TokenHistoryResource,
TokenResource,
)
from hathor.websocket import HathorAdminWebsocketFactory, WebsocketStatsResource
settings = HathorSettings()
cpu = get_cpu_profiler()
if args.prometheus:
kwargs: Dict[str, Any] = {'metrics': self.manager.metrics}
if args.data:
kwargs['path'] = os.path.join(args.data, 'prometheus')
else:
raise ValueError('To run prometheus exporter you must have a data path')
prometheus = PrometheusMetricsExporter(**kwargs)
prometheus.start()
if args.status:
# TODO get this from a file. How should we do with the factory?
root = Resource()
wallet_resource = Resource()
root.putChild(b'wallet', wallet_resource)
thin_wallet_resource = Resource()
root.putChild(b'thin_wallet', thin_wallet_resource)
contracts_resource = Resource()
wallet_resource.putChild(b'nano-contract', contracts_resource)
p2p_resource = Resource()
root.putChild(b'p2p', p2p_resource)
graphviz = Resource()
# XXX: reach the resource through /graphviz/ too, previously it was a leaf so this wasn't a problem
graphviz.putChild(b'', graphviz)
for fmt in ['dot', 'pdf', 'png', 'jpg']:
bfmt = fmt.encode('ascii')
graphviz.putChild(b'full.' + bfmt, GraphvizFullResource(self.manager, format=fmt))
graphviz.putChild(b'neighbours.' + bfmt, GraphvizNeighboursResource(self.manager, format=fmt))
resources = [
(b'status', StatusResource(self.manager), root),
(b'version', VersionResource(self.manager), root),
(b'create_tx', CreateTxResource(self.manager), root),
(b'decode_tx', DecodeTxResource(self.manager), root),
(b'validate_address', ValidateAddressResource(self.manager), root),
(b'push_tx',
PushTxResource(self.manager, args.max_output_script_size, args.allow_non_standard_script),
root),
(b'graphviz', graphviz, root),
(b'transaction', TransactionResource(self.manager), root),
(b'block_at_height', BlockAtHeightResource(self.manager), root),
(b'transaction_acc_weight', TransactionAccWeightResource(self.manager), root),
(b'dashboard_tx', DashboardTransactionResource(self.manager), root),
(b'profiler', ProfilerResource(self.manager), root),
(b'top', CPUProfilerResource(self.manager, cpu), root),
(b'mempool', MempoolResource(self.manager), root),
# mining
(b'mining', MiningResource(self.manager), root),
(b'getmininginfo', MiningInfoResource(self.manager), root),
(b'get_block_template', GetBlockTemplateResource(self.manager), root),
(b'submit_block', SubmitBlockResource(self.manager), root),
(b'tx_parents', TxParentsResource(self.manager), root),
# /thin_wallet
(b'address_history', AddressHistoryResource(self.manager), thin_wallet_resource),
(b'address_balance', AddressBalanceResource(self.manager), thin_wallet_resource),
(b'address_search', AddressSearchResource(self.manager), thin_wallet_resource),
(b'send_tokens', SendTokensThinResource(self.manager), thin_wallet_resource),
(b'token', TokenResource(self.manager), thin_wallet_resource),
(b'token_history', TokenHistoryResource(self.manager), thin_wallet_resource),
# /wallet/nano-contract
(b'match-value', NanoContractMatchValueResource(self.manager), contracts_resource),
(b'decode', NanoContractDecodeResource(self.manager), contracts_resource),
(b'execute', NanoContractExecuteResource(self.manager), contracts_resource),
# /p2p
(b'peers', AddPeersResource(self.manager), p2p_resource),
]
if args.enable_debug_api:
debug_resource = Resource()
root.putChild(b'_debug', debug_resource)
resources.extend([
(b'log', DebugLogResource(), debug_resource),
(b'raise', DebugRaiseResource(), debug_resource),
(b'reject', DebugRejectResource(), debug_resource),
(b'print', DebugPrintResource(), debug_resource),
])
if args.enable_crash_api:
crash_resource = Resource()
root.putChild(b'_crash', crash_resource)
resources.extend([
(b'exit', DebugCrashResource(), crash_resource),
(b'mess_around', DebugMessAroundResource(self.manager), crash_resource),
])
for url_path, resource, parent in resources:
parent.putChild(url_path, resource)
if self.manager.stratum_factory is not None:
from hathor.stratum.resources import MiningStatsResource
root.putChild(b'miners', MiningStatsResource(self.manager))
with_wallet_api = bool(self.wallet and args.wallet_enable_api)
if with_wallet_api:
wallet_resources = (
# /wallet
(b'balance', BalanceResource(self.manager), wallet_resource),
(b'history', HistoryResource(self.manager), wallet_resource),
(b'address', AddressResource(self.manager), wallet_resource),
(b'send_tokens', SendTokensResource(self.manager), wallet_resource),
(b'sign_tx', SignTxResource(self.manager), wallet_resource),
(b'unlock', UnlockWalletResource(self.manager), wallet_resource),
(b'lock', LockWalletResource(self.manager), wallet_resource),
(b'state', StateWalletResource(self.manager), wallet_resource),
)
for url_path, resource, parent in wallet_resources:
parent.putChild(url_path, resource)
# Websocket resource
assert self.manager.tx_storage.indexes is not None
ws_factory = HathorAdminWebsocketFactory(metrics=self.manager.metrics,
address_index=self.manager.tx_storage.indexes.addresses)
ws_factory.start()
root.putChild(b'ws', WebSocketResource(ws_factory))
# Mining websocket resource
mining_ws_factory = MiningWebsocketFactory(self.manager)
root.putChild(b'mining_ws', WebSocketResource(mining_ws_factory))
ws_factory.subscribe(self.manager.pubsub)
# Websocket stats resource
root.putChild(b'websocket_stats', WebsocketStatsResource(ws_factory))
real_root = Resource()
real_root.putChild(settings.API_VERSION_PREFIX.encode('ascii'), root)
from hathor.profiler.site import SiteProfiler
status_server = SiteProfiler(real_root)
reactor.listenTCP(args.status, status_server)
self.log.info('with status', listen=args.status, with_wallet_api=with_wallet_api)
# Set websocket factory in metrics
self.manager.metrics.websocket_factory = ws_factory
def register_signal_handlers(self, args: Namespace) -> None:
"""Register signal handlers."""
import signal
sigusr1 = getattr(signal, 'SIGUSR1', None)
if sigusr1 is not None:
# USR1 is avaiable in this OS.
signal.signal(sigusr1, self.signal_usr1_handler)
def signal_usr1_handler(self, sig: int, frame: Any) -> None:
"""Called when USR1 signal is received."""
self.log.warn('USR1 received. Killing all connections...')
if self.manager and self.manager.connections:
self.manager.connections.disconnect_all_peers(force=True)
def check_unsafe_arguments(self, args: Namespace) -> None:
unsafe_args_found = []
for arg_cmdline, arg_test_fn in self.UNSAFE_ARGUMENTS:
if arg_test_fn(args):
unsafe_args_found.append(arg_cmdline)
if args.unsafe_mode is None:
if unsafe_args_found:
message = [
'You need to enable --unsafe-mode to run with these arguments.',
'',
'The following argument require unsafe mode:',
]
for arg_cmdline in unsafe_args_found:
message.append(arg_cmdline)
message.extend([
'',
'Never enable UNSAFE MODE in a production environment.'
])
self.log.critical('\n'.join(message))
sys.exit(-1)
else:
fail = False
message = [
'UNSAFE MODE IS ENABLED',
'',
'********************************************************',
'********************************************************',
'',
'UNSAFE MODE IS ENABLED',
'',
'You should never use --unsafe-mode in production environments.',
'',
]
from hathor.conf import HathorSettings
settings = HathorSettings()
if args.unsafe_mode != settings.NETWORK_NAME:
message.extend([
f'Unsafe mode enabled for wrong network ({args.unsafe_mode} != {settings.NETWORK_NAME}).',
'',
])
fail = True
is_local_network = True
if settings.NETWORK_NAME == 'mainnet':
is_local_network = False
elif settings.NETWORK_NAME.startswith('testnet'):
is_local_network = False
if not is_local_network:
message.extend([
f'You should not enable unsafe mode on {settings.NETWORK_NAME} unless you know what you are doing',
'',
])
if not unsafe_args_found:
message.extend([
'--unsafe-mode is not needed because you have not enabled any unsafe feature.',
'',
'Remove --unsafe-mode and try again.',
])
fail = True
else:
message.append('You have enabled the following features:')
for arg_cmdline in unsafe_args_found:
message.append(arg_cmdline)
message.extend([
'',
'********************************************************',
'********************************************************',
'',
])
self.log.critical('\n'.join(message))
if fail:
sys.exit(-1)
def __init__(self, *, argv=None):
if argv is None:
import sys
argv = sys.argv[1:]
self.parser = self.create_parser()
args = self.parse_args(argv)
if args.testnet:
if not os.environ.get('HATHOR_CONFIG_FILE'):
os.environ['HATHOR_CONFIG_FILE'] = 'hathor.conf.testnet'
self.prepare(args)
self.register_signal_handlers(args)
def parse_args(self, argv: List[str]) -> Namespace:
return self.parser.parse_args(argv)
def run(self) -> None:
reactor.run()
class BaseTransactionStorageTest(unittest.TestCase):
__test__ = False
def setUp(self, tx_storage, reactor=None):
from hathor.manager import HathorManager
if not reactor:
self.reactor = Clock()
else:
self.reactor = reactor
self.reactor.advance(time.time())
self.tx_storage = tx_storage
assert tx_storage.first_timestamp > 0
tx_storage._manually_initialize()
self.genesis = self.tx_storage.get_all_genesis()
self.genesis_blocks = [tx for tx in self.genesis if tx.is_block]
self.genesis_txs = [tx for tx in self.genesis if not tx.is_block]
self.tmpdir = tempfile.mkdtemp()
wallet = Wallet(directory=self.tmpdir)
wallet.unlock(b'teste')
self.manager = HathorManager(self.reactor,
tx_storage=self.tx_storage,
wallet=wallet)
self.tx_storage.indexes.enable_address_index(self.manager.pubsub)
self.tx_storage.indexes.enable_tokens_index()
block_parents = [
tx.hash for tx in chain(self.genesis_blocks, self.genesis_txs)
]
output = TxOutput(200, P2PKH.create_output_script(BURN_ADDRESS))
self.block = Block(timestamp=MIN_TIMESTAMP,
weight=12,
outputs=[output],
parents=block_parents,
nonce=100781,
storage=tx_storage)
self.block.resolve()
self.block.verify()
self.block.get_metadata().validation = ValidationState.FULL
tx_parents = [tx.hash for tx in self.genesis_txs]
tx_input = TxInput(
tx_id=self.genesis_blocks[0].hash,
index=0,
data=bytes.fromhex(
'46304402203470cb9818c9eb842b0c433b7e2b8aded0a51f5903e971649e870763d0266a'
'd2022049b48e09e718c4b66a0f3178ef92e4d60ee333d2d0e25af8868acf5acbb35aaa583'
'056301006072a8648ce3d020106052b8104000a034200042ce7b94cba00b654d4308f8840'
'7345cacb1f1032fb5ac80407b74d56ed82fb36467cb7048f79b90b1cf721de57e942c5748'
'620e78362cf2d908e9057ac235a63'))
self.tx = Transaction(
timestamp=MIN_TIMESTAMP + 2,
weight=10,
nonce=932049,
inputs=[tx_input],
outputs=[output],
tokens=[
bytes.fromhex(
'0023be91834c973d6a6ddd1a0ae411807b7c8ef2a015afb5177ee64b666ce602'
)
],
parents=tx_parents,
storage=tx_storage)
self.tx.resolve()
self.tx.get_metadata().validation = ValidationState.FULL
# Disable weakref to test the internal methods. Otherwise, most methods return objects from weakref.
self.tx_storage._disable_weakref()
self.tx_storage.enable_lock()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_genesis_ref(self):
# Enable weakref to this test only.
self.tx_storage._enable_weakref()
genesis_set = set(self.tx_storage.get_all_genesis())
for tx in genesis_set:
tx2 = self.tx_storage.get_transaction(tx.hash)
self.assertTrue(tx is tx2)
from hathor.transaction.genesis import _get_genesis_transactions_unsafe
genesis_from_settings = _get_genesis_transactions_unsafe(None)
for tx in genesis_from_settings:
tx2 = self.tx_storage.get_transaction(tx.hash)
self.assertTrue(tx is not tx2)
for tx3 in genesis_set:
self.assertTrue(tx is not tx3)
if tx2 == tx3:
self.assertTrue(tx2 is tx3)
def test_genesis(self):
self.assertEqual(1, len(self.genesis_blocks))
self.assertEqual(2, len(self.genesis_txs))
for tx in self.genesis:
tx.verify()
for tx in self.genesis:
tx2 = self.tx_storage.get_transaction(tx.hash)
self.assertEqual(tx, tx2)
self.assertTrue(self.tx_storage.transaction_exists(tx.hash))
def test_get_empty_merklee_tree(self):
# We use `first_timestamp - 1` to ensure that the merkle tree will be empty.
self.tx_storage.get_merkle_tree(self.tx_storage.first_timestamp - 1)
def test_first_timestamp(self):
self.assertEqual(self.tx_storage.first_timestamp,
min(x.timestamp for x in self.genesis))
def test_storage_basic(self):
self.assertEqual(1, self.tx_storage.get_block_count())
self.assertEqual(2, self.tx_storage.get_tx_count())
self.assertEqual(3, self.tx_storage.get_count_tx_blocks())
block_parents_hash = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(1, len(block_parents_hash))
self.assertEqual(block_parents_hash, [self.genesis_blocks[0].hash])
tx_parents_hash = [x.data for x in self.tx_storage.get_tx_tips()]
self.assertEqual(2, len(tx_parents_hash))
self.assertEqual(set(tx_parents_hash),
{self.genesis_txs[0].hash, self.genesis_txs[1].hash})
def test_storage_basic_v2(self):
self.assertEqual(1, self.tx_storage.get_block_count())
self.assertEqual(2, self.tx_storage.get_tx_count())
self.assertEqual(3, self.tx_storage.get_count_tx_blocks())
block_parents_hash = self.tx_storage.get_best_block_tips()
self.assertEqual(1, len(block_parents_hash))
self.assertEqual(block_parents_hash, [self.genesis_blocks[0].hash])
tx_parents_hash = self.manager.get_new_tx_parents()
self.assertEqual(2, len(tx_parents_hash))
self.assertEqual(set(tx_parents_hash),
{self.genesis_txs[0].hash, self.genesis_txs[1].hash})
def validate_save(self, obj):
self.tx_storage.save_transaction(obj, add_to_indexes=True)
loaded_obj1 = self.tx_storage.get_transaction(obj.hash)
self.assertTrue(self.tx_storage.transaction_exists(obj.hash))
self.assertEqual(obj, loaded_obj1)
self.assertEqual(len(obj.get_funds_struct()),
len(loaded_obj1.get_funds_struct()))
self.assertEqual(bytes(obj), bytes(loaded_obj1))
self.assertEqual(obj.to_json(), loaded_obj1.to_json())
self.assertEqual(obj.is_block, loaded_obj1.is_block)
# Testing add and remove from cache
if self.tx_storage.with_index:
if obj.is_block:
self.assertTrue(obj.hash in self.tx_storage.indexes.block_tips.
tx_last_interval)
else:
self.assertTrue(obj.hash in self.tx_storage.indexes.tx_tips.
tx_last_interval)
self.tx_storage.del_from_indexes(obj)
if self.tx_storage.with_index:
if obj.is_block:
self.assertFalse(obj.hash in self.tx_storage.indexes.
block_tips.tx_last_interval)
else:
self.assertFalse(obj.hash in self.tx_storage.indexes.tx_tips.
tx_last_interval)
self.tx_storage.add_to_indexes(obj)
if self.tx_storage.with_index:
if obj.is_block:
self.assertTrue(obj.hash in self.tx_storage.indexes.block_tips.
tx_last_interval)
else:
self.assertTrue(obj.hash in self.tx_storage.indexes.tx_tips.
tx_last_interval)
def test_save_block(self):
self.validate_save(self.block)
def test_save_tx(self):
self.validate_save(self.tx)
def test_save_token_creation_tx(self):
tx = create_tokens(self.manager, propagate=False)
tx.get_metadata().validation = ValidationState.FULL
self.validate_save(tx)
def _validate_not_in_index(self, tx, index):
tips = index.tips_index[self.tx.timestamp]
self.assertNotIn(self.tx.hash, [x.data for x in tips])
self.assertNotIn(self.tx.hash, index.tips_index.tx_last_interval)
self.assertIsNone(index.txs_index.find_tx_index(tx))
def _test_remove_tx_or_block(self, tx):
self.validate_save(tx)
self.tx_storage.remove_transaction(tx)
with self.assertRaises(TransactionDoesNotExist):
self.tx_storage.get_transaction(tx.hash)
if hasattr(self.tx_storage, 'all_index'):
self._validate_not_in_index(tx, self.tx_storage.all_index)
if tx.is_block:
if hasattr(self.tx_storage, 'block_index'):
self._validate_not_in_index(tx, self.tx_storage.block_index)
else:
if hasattr(self.tx_storage, 'tx_index'):
self._validate_not_in_index(tx, self.tx_storage.tx_index)
# Check wallet index.
addresses_index = self.tx_storage.indexes.addresses
addresses = tx.get_related_addresses()
for address in addresses:
self.assertNotIn(tx.hash,
addresses_index.get_from_address(address))
# TODO Check self.tx_storage.tokens_index
# Try to remove twice. It is supposed to do nothing.
self.tx_storage.remove_transaction(tx)
def test_remove_tx(self):
self._test_remove_tx_or_block(self.tx)
def test_remove_block(self):
self._test_remove_tx_or_block(self.block)
def test_shared_memory(self):
# Enable weakref to this test only.
self.tx_storage._enable_weakref()
self.validate_save(self.block)
self.validate_save(self.tx)
for tx in [self.tx, self.block]:
# just making sure, if it is genesis the test is wrong
self.assertFalse(tx.is_genesis)
# load transactions twice
tx1 = self.tx_storage.get_transaction(tx.hash)
tx2 = self.tx_storage.get_transaction(tx.hash)
# naturally they should be equal, but this time so do the objects
self.assertTrue(tx1 == tx2)
self.assertTrue(tx1 is tx2)
meta1 = tx1.get_metadata()
meta2 = tx2.get_metadata()
# and naturally the metadata too
self.assertTrue(meta1 == meta2)
self.assertTrue(meta1 is meta2)
def test_get_wrong_tx(self):
hex_error = bytes.fromhex(
'00001c5c0b69d13b05534c94a69b2c8272294e6b0c536660a3ac264820677024')
with self.assertRaises(TransactionDoesNotExist):
self.tx_storage.get_transaction(hex_error)
def test_save_metadata(self):
# Saving genesis metadata
self.tx_storage.save_transaction(self.genesis_txs[0],
only_metadata=True)
tx = self.block
# First we save to the storage
self.tx_storage.save_transaction(tx)
metadata = tx.get_metadata()
metadata.spent_outputs[1].append(self.genesis_blocks[0].hash)
random_tx = bytes.fromhex(
'0000222e64683b966b4268f387c269915cc61f6af5329823a93e3696cb0f2222')
metadata.children.append(random_tx)
self.tx_storage.save_transaction(tx, only_metadata=True)
tx2 = self.tx_storage.get_transaction(tx.hash)
metadata2 = tx2.get_metadata()
self.assertEqual(metadata, metadata2)
total = 0
for tx in self.tx_storage.get_all_transactions():
total += 1
self.assertEqual(total, 4)
def test_storage_new_blocks(self):
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(tip_blocks, [self.genesis_blocks[0].hash])
block1 = self._add_new_block()
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(tip_blocks, [block1.hash])
block2 = self._add_new_block()
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(tip_blocks, [block2.hash])
# Block3 has the same parents as block2.
block3 = self._add_new_block(parents=block2.parents)
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(set(tip_blocks), {block2.hash, block3.hash})
# Re-generate caches to test topological sort.
self.tx_storage._manually_initialize()
tip_blocks = [x.data for x in self.tx_storage.get_block_tips()]
self.assertEqual(set(tip_blocks), {block2.hash, block3.hash})
def test_token_list(self):
tx = self.tx
self.validate_save(tx)
# 2 token uids
tx.tokens.append(
bytes.fromhex(
'00001c5c0b69d13b05534c94a69b2c8272294e6b0c536660a3ac264820677024'
))
tx.resolve()
self.validate_save(tx)
# no tokens
tx.tokens = []
tx.resolve()
self.validate_save(tx)
def _add_new_block(self, parents=None):
block = self.manager.generate_mining_block()
block.data = b'Testing, testing, 1, 2, 3... testing, testing...'
if parents is not None:
block.parents = parents
block.weight = 10
self.assertTrue(block.resolve())
block.verify()
self.manager.propagate_tx(block, fails_silently=False)
self.reactor.advance(5)
return block
def test_topological_sort(self):
_set_test_mode(TestMode.TEST_ALL_WEIGHT)
_total = 0
blocks = add_new_blocks(self.manager, 1, advance_clock=1)
_total += len(blocks)
blocks = add_blocks_unlock_reward(self.manager)
_total += len(blocks)
add_new_transactions(self.manager, 1, advance_clock=1)
total = 0
for tx in self.tx_storage._topological_sort():
total += 1
# added blocks + genesis txs + added tx
self.assertEqual(total, _total + 3 + 1)
def test_get_best_block_weight(self):
block = self._add_new_block()
weight = self.tx_storage.get_weight_best_block()
self.assertEqual(block.weight, weight)
@inlineCallbacks
def test_concurrent_access(self):
self.tx_storage.save_transaction(self.tx)
self.tx_storage._enable_weakref()
def handle_error(err):
self.fail(
'Error resolving concurrent access deferred. {}'.format(err))
deferreds = []
for i in range(5):
d = deferToThread(self.tx_storage.get_transaction, self.tx.hash)
d.addErrback(handle_error)
deferreds.append(d)
self.reactor.advance(3)
yield gatherResults(deferreds)
self.tx_storage._disable_weakref()
def test_full_verification_attribute(self):
self.assertFalse(self.tx_storage.is_running_full_verification())
self.tx_storage.start_full_verification()
self.assertTrue(self.tx_storage.is_running_full_verification())
self.tx_storage.finish_full_verification()
self.assertFalse(self.tx_storage.is_running_full_verification())
def test_key_value_attribute(self):
attr = 'test'
val = 'a'
# Try to get a key that does not exist
self.assertIsNone(self.tx_storage.get_value(attr))
# Try to remove this key that does not exist
self.tx_storage.remove_value(attr)
# Add the key/value
self.tx_storage.add_value(attr, val)
# Get correct value
self.assertEqual(self.tx_storage.get_value(attr), val)
# Remove the key
self.tx_storage.remove_value(attr)
# Key should not exist again
self.assertIsNone(self.tx_storage.get_value(attr))
def prepare(self, args: Namespace) -> None:
import hathor
from hathor.cli.util import check_or_exit
from hathor.conf import HathorSettings
from hathor.conf.get_settings import get_settings_module
from hathor.daa import TestMode, _set_test_mode
from hathor.manager import HathorManager
from hathor.p2p.peer_discovery import BootstrapPeerDiscovery, DNSPeerDiscovery
from hathor.p2p.peer_id import PeerId
from hathor.p2p.utils import discover_hostname
from hathor.transaction import genesis
from hathor.transaction.storage import (
TransactionCacheStorage,
TransactionCompactStorage,
TransactionMemoryStorage,
TransactionRocksDBStorage,
TransactionStorage,
)
from hathor.wallet import HDWallet, Wallet
settings = HathorSettings()
settings_module = get_settings_module() # only used for logging its location
self.log = logger.new()
from setproctitle import setproctitle
setproctitle('{}hathor-core'.format(args.procname_prefix))
if args.recursion_limit:
sys.setrecursionlimit(args.recursion_limit)
else:
sys.setrecursionlimit(5000)
try:
import resource
except ModuleNotFoundError:
pass
else:
(nofile_soft, _) = resource.getrlimit(resource.RLIMIT_NOFILE)
if nofile_soft < 256:
print('Maximum number of open file descriptors is too low. Minimum required is 256.')
sys.exit(-2)
if not args.peer:
peer_id = PeerId()
else:
data = json.load(open(args.peer, 'r'))
peer_id = PeerId.create_from_json(data)
python = f'{platform.python_version()}-{platform.python_implementation()}'
self.check_unsafe_arguments(args)
self.log.info(
'hathor-core v{hathor}',
hathor=hathor.__version__,
pid=os.getpid(),
genesis=genesis.GENESIS_HASH.hex()[:7],
my_peer_id=str(peer_id.id),
python=python,
platform=platform.platform(),
settings=settings_module.__file__,
)
def create_wallet():
if args.wallet == 'hd':
kwargs = {
'words': args.words,
}
if args.passphrase:
wallet_passphrase = getpass.getpass(prompt='HD Wallet passphrase:')
kwargs['passphrase'] = wallet_passphrase.encode()
if args.data:
kwargs['directory'] = args.data
return HDWallet(**kwargs)
elif args.wallet == 'keypair':
print('Using KeyPairWallet')
if args.data:
wallet = Wallet(directory=args.data)
else:
wallet = Wallet()
wallet.flush_to_disk_interval = 5 # seconds
if args.unlock_wallet:
wallet_passwd = getpass.getpass(prompt='Wallet password:'******'Invalid type for wallet')
tx_storage: TransactionStorage
if args.memory_storage:
check_or_exit(not args.data, '--data should not be used with --memory-storage')
# if using MemoryStorage, no need to have cache
tx_storage = TransactionMemoryStorage()
assert not args.x_rocksdb_indexes, 'RocksDB indexes require RocksDB data'
self.log.info('with storage', storage_class=type(tx_storage).__name__)
elif args.json_storage:
check_or_exit(args.data, '--data is expected')
assert not args.x_rocksdb_indexes, 'RocksDB indexes require RocksDB data'
tx_storage = TransactionCompactStorage(path=args.data, with_index=(not args.cache))
else:
check_or_exit(args.data, '--data is expected')
if args.rocksdb_storage:
self.log.warn('--rocksdb-storage is now implied, no need to specify it')
cache_capacity = args.rocksdb_cache
use_memory_indexes = not args.x_rocksdb_indexes
tx_storage = TransactionRocksDBStorage(path=args.data, with_index=(not args.cache),
cache_capacity=cache_capacity,
use_memory_indexes=use_memory_indexes)
self.log.info('with storage', storage_class=type(tx_storage).__name__, path=args.data)
if args.cache:
check_or_exit(not args.memory_storage, '--cache should not be used with --memory-storage')
tx_storage = TransactionCacheStorage(tx_storage, reactor)
if args.cache_size:
tx_storage.capacity = args.cache_size
if args.cache_interval:
tx_storage.interval = args.cache_interval
self.log.info('with cache', capacity=tx_storage.capacity, interval=tx_storage.interval)
self.tx_storage = tx_storage
self.log.info('with indexes', indexes_class=type(tx_storage.indexes).__name__)
if args.wallet:
self.wallet = create_wallet()
self.log.info('with wallet', wallet=self.wallet, path=args.data)
else:
self.wallet = None
if args.hostname and args.auto_hostname:
print('You cannot use --hostname and --auto-hostname together.')
sys.exit(-1)
if not args.auto_hostname:
hostname = args.hostname
else:
print('Trying to discover your hostname...')
hostname = discover_hostname()
if not hostname:
print('Aborting because we could not discover your hostname.')
print('Try again or run without --auto-hostname.')
sys.exit(-1)
print('Hostname discovered and set to {}'.format(hostname))
network = settings.NETWORK_NAME
enable_sync_v1 = not args.x_sync_v2_only
enable_sync_v2 = args.x_sync_v2_only or args.x_sync_bridge
self.manager = HathorManager(
reactor,
peer_id=peer_id,
network=network,
hostname=hostname,
tx_storage=self.tx_storage,
wallet=self.wallet,
wallet_index=args.wallet_index,
stratum_port=args.stratum,
ssl=True,
checkpoints=settings.CHECKPOINTS,
enable_sync_v1=enable_sync_v1,
enable_sync_v2=enable_sync_v2,
soft_voided_tx_ids=set(settings.SOFT_VOIDED_TX_IDS),
)
if args.allow_mining_without_peers:
self.manager.allow_mining_without_peers()
if args.x_localhost_only:
self.manager.connections.localhost_only = True
dns_hosts = []
if settings.BOOTSTRAP_DNS:
dns_hosts.extend(settings.BOOTSTRAP_DNS)
if args.dns:
dns_hosts.extend(args.dns)
if dns_hosts:
self.manager.add_peer_discovery(DNSPeerDiscovery(dns_hosts))
if args.bootstrap:
self.manager.add_peer_discovery(BootstrapPeerDiscovery(args.bootstrap))
if args.test_mode_tx_weight:
_set_test_mode(TestMode.TEST_TX_WEIGHT)
if self.wallet:
self.wallet.test_mode = True
if args.x_full_verification:
self.manager._full_verification = True
if args.x_fast_init_beta:
self.log.warn('--x-fast-init-beta is now the default, no need to specify it')
for description in args.listen:
self.manager.add_listen_address(description)
self.start_manager(args)
self.register_resources(args)
