class ConnectionMonitor(gevent.Greenlet):
"monitors the connection by sending pings and checking pongs"
ping_interval = 15.
response_delay_threshold = 120. # FIXME, apply msg takes too long
max_samples = 1000
log = slogging.get_logger('p2p.ctxmonitor')
def __init__(self, proto):
self.log.debug('init')
assert isinstance(proto, P2PProtocol)
self.proto = proto
self.samples = collections.deque(maxlen=self.max_samples)
self.last_response = self.last_request = time.time()
super(ConnectionMonitor, self).__init__()
# track responses
self.proto.receive_pong_callbacks.append(self.track_response)
self.proto.receive_hello_callbacks.append(
lambda p, **kargs: self.start())
def track_response(self, proto):
self.last_response = time.time()
self.samples.appendleft(self.last_response - self.last_request)
def latency(self, num_samples=max_samples):
num_samples = min(num_samples, len(self.samples))
return sum(
self.samples[i]
for i in range(num_samples)) / num_samples if num_samples else 1
def _run(self):
self.log.debug('started', monitor=self)
while True:
self.log.debug('pinging', monitor=self)
self.proto.send_ping()
now = self.last_request = time.time()
gevent.sleep(self.ping_interval)
self.log.debug('latency',
peer=self.proto,
latency='%.3f' % self.latency())
if now - self.last_response > self.response_delay_threshold:
self.log.debug('unresponsive_peer', monitor=self)
self.proto.peer.report_error('not responding to ping')
self.proto.stop()
self.kill()
def stop(self):
self.log.debug('stopped', monitor=self)
self.kill()
import socket
import atexit
import time
from gevent.server import StreamServer
from gevent.socket import create_connection, timeout
from service import WiredService
from protocol import BaseProtocol
from p2p_protocol import P2PProtocol
from discovery import NodeDiscovery
import kademlia
from peer import Peer
import crypto
import utils
import slogging
log = slogging.get_logger('p2p.peermgr')
class PeerManager(WiredService):
"""
todo:
connects new peers if there are too few
selects peers based on a DHT
keeps track of peer reputation
saves/loads peers (rather discovery buckets) to disc
connection strategy
for service which requires peers
while num peers > min_num_peers:
gen random id
# https://github.com/ethereum/go-ethereum/wiki/Blockpool
import gevent
import rlp
from rlp import sedes
from multiplexer import Packet
from service import WiredService
import slogging
log = slogging.get_logger('protocol')
class ProtocolError(Exception):
pass
class SubProtocolError(ProtocolError):
pass
class BaseProtocol(gevent.Greenlet):
"""
A protocol mediates between the network and the service.
It implements a collection of commands.
For each command X the following methods are created at initialization:
- packet = protocol.create_X(*args, **kargs)
- protocol.send_X(*args, **kargs) is a shortcut for:
protocol.send_packet(protocol.create_X(*args, **kargs))
- protocol._receive_X(data)
"""
import time
import struct
import gevent
import gevent.socket
from devp2p import crypto
import rlp
from devp2p import utils
from devp2p import kademlia
from service import BaseService
from gevent.server import DatagramServer
import slogging
import ipaddress
log = slogging.get_logger('discovery')
class DefectiveMessage(Exception):
pass
class InvalidSignature(DefectiveMessage):
pass
class PacketExpired(DefectiveMessage):
pass
class Address(object):
from protocol import BaseProtocol
from p2p_protocol import P2PProtocol
import kademlia
from peer import Peer
import crypto
import utils
from devp2p import discovery
import slogging
import sys
#log = slogging.get_logger('p2p.peermgr')
slogging.configure(':DEBUG')
root_logger = slogging.getLogger()
if root_logger.handlers == []:
root_logger.addHandler(slogging.logging.StreamHandler(sys.stderr))
log = slogging.get_logger('p2p')
from collections import defaultdict
class ETHProtocol(BaseProtocol):
"""
DEV Ethereum Wire Protocol
https://github.com/ethereum/wiki/wiki/Ethereum-Wire-Protocol
https://github.com/ethereum/go-ethereum/blob/develop/eth/protocol.go#L15
"""
protocol_id = 1
network_id = 0
max_cmd_id = 15 # FIXME
name = 'eth'
version = 63
import gevent
from collections import OrderedDict
from protocol import BaseProtocol
from protocol import decode_packet_header, header_length
import slogging
log = slogging.get_logger('peer')
class QueueWorker(gevent.Greenlet):
# FIXME we need to queue send messages
def __init__(self, queue):
self.queue = queue
super(QueueWorker, self).__init__()
def _run(self):
self.running = True
while self.running:
msg = self.queue.get() # block call
print('queue:', msg)
class Peer(gevent.Greenlet):
"""
After creation:
register peer protocol
send hello & encryption
receive hello & derive session key
register in common protocols
receive data
https://github.com/ethereum/go-ethereum/wiki/RLPx-----Node-Discovery-Protocol
"""
import time
import gevent
import gevent.socket
from devp2p import crypto
import rlp
from devp2p import utils
from devp2p import kademlia
from service import BaseService
from gevent.server import DatagramServer
import slogging
import ipaddress
log = slogging.get_logger('p2p.discovery')
class DefectiveMessage(Exception):
pass
class InvalidSignature(DefectiveMessage):
pass
class PacketExpired(DefectiveMessage):
pass
class Address(object):
# https://github.com/ethereum/go-ethereum/wiki/Blockpool
import gevent
import rlp
from rlp import sedes
from multiplexer import Packet
from service import WiredService
import slogging
log = slogging.get_logger('protocol')
class ProtocolError(Exception):
pass
class SubProtocolError(ProtocolError):
pass
class BaseProtocol(gevent.Greenlet):
"""
A protocol mediates between the network and the service.
It implements a collection of commands.
For each command X the following methods are created at initialization:
- packet = protocol.create_X(*args, **kargs)
- protocol.send_X(*args, **kargs) is a shortcut for:
protocol.send_packet(protocol.create_X(*args, **kargs))
- protocol._receive_X(data)
on protocol.receive_packet, the packet is deserialized according to the command.structure
request timeouts are 300ms, and
the idle bucket-refresh interval is 3600 seconds.
Aside from the previously described exclusions, node discovery closely follows system
and protocol described by Maymounkov and Mazieres.
"""
import operator
import random
import time
from functools import total_ordering
import slogging
from crypto import sha3
from utils import big_endian_to_int
log = slogging.get_logger('p2p.discovery.kademlia')
k_b = 8 # 8 bits per hop
k_bucket_size = 16
k_request_timeout = 3 * 300 / 1000. # timeout of message round trips
k_idle_bucket_refresh_interval = 3600 # ping all nodes in bucket if bucket was idle
k_find_concurrency = 3 # parallel find node lookups
k_pubkey_size = 512
k_id_size = 256
k_max_node_id = 2**k_id_size - 1
def random_nodeid():
return random.randint(0, k_max_node_id)
and 8 bits per hop (denoted b in Kademlia) for routing.
The eviction check interval is 75 milliseconds,
request timeouts are 300ms, and
the idle bucket-refresh interval is 3600 seconds.
Aside from the previously described exclusions, node discovery closely follows system
and protocol described by Maymounkov and Mazieres.
"""
from utils import big_endian_to_int
from crypto import sha3
import operator
import time
import random
import slogging
log = slogging.get_logger('p2p.discovery.kademlia')
k_b = 8 # 8 bits per hop
k_bucket_size = 16
k_request_timeout = 3 * 300 / 1000. # timeout of message round trips
k_idle_bucket_refresh_interval = 3600 # ping all nodes in bucket if bucket was idle
k_find_concurrency = 3 # parallel find node lookups
k_pubkey_size = 512
k_id_size = 256
k_max_node_id = 2 ** k_id_size - 1
def random_nodeid():
return random.randint(0, k_max_node_id)
import gevent
import operator
from collections import OrderedDict
from protocol import BaseProtocol
from p2p_protocol import P2PProtocol
from service import WiredService
import multiplexer
from muxsession import MultiplexedSession
import slogging
import gevent.socket
import rlpxcipher
log = slogging.get_logger('peer')
class Peer(gevent.Greenlet):
remote_client_version = ''
wait_read_timeout = 0.001
def __init__(self, peermanager, connection, remote_pubkey=None): # FIXME node vs remote_pubkey
super(Peer, self).__init__()
self.is_stopped = False
self.peermanager = peermanager
self.connection = connection
self.config = peermanager.config
self.protocols = OrderedDict()
log.debug('peer init', peer=self)
# create multiplexed encrypted session
privkey = self.config['node']['privkey_hex'].decode('hex')
peer=self.proto,
latency='%.3f' % self.latency())
if now - self.last_response > self.response_delay_threshold:
self.log.debug('unresponsive_peer', monitor=self)
self.proto.peer.report_error('not responding to ping')
self.proto.stop()
self.kill()
def stop(self):
self.log.debug('stopped', monitor=self)
self.kill()
########################################
log = slogging.get_logger('protocol.p2p')
class P2PProtocol(BaseProtocol):
"""
DEV P2P Wire Protocol
https://github.com/ethereum/wiki/wiki/%C3%90%CE%9EVp2p-Wire-Protocol
"""
protocol_id = 0
name = 'p2p'
version = 3
max_cmd_id = 15
def __init__(self, peer, service):
# required by P2PProtocol
self.config = peer.config
and 8 bits per hop (denoted b in Kademlia) for routing.
The eviction check interval is 75 milliseconds,
request timeouts are 300ms, and
the idle bucket-refresh interval is 3600 seconds.
Aside from the previously described exclusions, node discovery closely follows system
and protocol described by Maymounkov and Mazieres.
"""
from utils import big_endian_to_int
from crypto import sha3
import operator
import time
import random
import slogging
log = slogging.get_logger('kademlia')
k_b = 8 # 8 bits per hop
k_bucket_size = 16
k_request_timeout = 3 * 300 / 1000. # timeout of message round trips
k_idle_bucket_refresh_interval = 3600 # ping all nodes in bucket if bucket was idle
k_find_concurrency = 3 # parallel find node lookups
k_pubkey_size = 512
k_id_size = 512
k_max_node_id = 2**k_id_size - 1
class Node(object):
def __init__(self, pubkey):
assert len(pubkey) == 64 and isinstance(pubkey, str)
import rlp
from utils import idec
from utils import ienc4
from utils import recursive_int_to_big_endian
from slogging import get_logger
log = get_logger('serialization')
def lrlp_decode(data):
"always return a list"
d = rlp.decode(data)
if isinstance(d, str):
d = [d]
return d
class Serializer(object):
SYNCHRONIZATION_TOKEN = 0x22400891
disconnect_reasons_map = dict(
(('Disconnect requested', 0x00), ('TCP sub-system error', 0x01),
('Bad protocol', 0x02), ('Useless peer', 0x03),
('Too many peers', 0x04), ('Already connected', 0x05),
('Wrong genesis block', 0x06), ('Incompatible network protocols',
0x07), ('Client quitting', 0x08)))
disconnect_reasons_map_by_id = \
dict((v, k) for k, v in disconnect_reasons_map.items())
@classmethod
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# https://github.com/robnewton/JSON-RPC-Browser
import gevent
import gevent.wsgi
import gevent.queue
from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
from tinyrpc.transports.wsgi import WsgiServerTransport
from tinyrpc.server.gevent import RPCServerGreenlets
from tinyrpc.dispatch import RPCDispatcher
from service import BaseService
import slogging
log = slogging.get_logger('jsonrpc')
class JSONRPCServer(BaseService):
name = 'jsonrpc'
def __init__(self, app):
log.debug('initializing JSONRPCServer')
BaseService.__init__(self, app)
self.app = app
self.dispatcher = RPCDispatcher()
transport = WsgiServerTransport(queue_class=gevent.queue.Queue)
# start wsgi server as a background-greenlet
self.wsgi_server = gevent.wsgi.WSGIServer(('127.0.0.1', 5000),
transport.handle)
# -*- coding: utf-8 -*-
import time
import gevent
import gevent.socket
from socket import AF_INET, AF_INET6
from devp2p import crypto
import rlp
from devp2p import utils
from devp2p import kademlia
from service import BaseService
from gevent.server import DatagramServer
import slogging
import ipaddress
log = slogging.get_logger('p2p.discovery')
class DefectiveMessage(Exception):
pass
class InvalidSignature(DefectiveMessage):
pass
class WrongMAC(DefectiveMessage):
pass
class PacketExpired(DefectiveMessage):
from UserDict import IterableUserDict
from service import BaseService
from slogging import get_logger
import utils
import crypto
from devp2p import __version__
log = get_logger('app')
class BaseApp(object):
default_config = dict(client_version='pydevp2p {}'.format(__version__),
deactivated_services=[])
def __init__(self, config=default_config):
self.config = utils.update_config_with_defaults(config, self.default_config)
self.services = IterableUserDict()
def register_service(self, service):
"""
registeres protocol with app, which will be accessible as
app.services.<protocol.name> (e.g. app.services.p2p or app.services.eth)
"""
assert isinstance(service, BaseService)
assert service.name not in self.services
log.info('registering service', service=service.name)
self.services[service.name] = service
setattr(self.services, service.name, service)
def deregister_service(self, service):
assert isinstance(service, BaseService)
from UserDict import IterableUserDict
from service import BaseService
from slogging import get_logger
import crypto
log = get_logger('app')
class BaseApp(object):
def __init__(self, config):
self.config = config
self.services = IterableUserDict()
def register_service(self, service):
"""
registeres protocol with peer, which will be accessible as
peer.<protocol.name> (e.g. peer.p2p or peer.eth)
"""
assert isinstance(service, BaseService)
assert service.name not in self.services
log.info('registering service', service=service.name)
self.services[service.name] = service
setattr(self.services, service.name, service)
def deregister_service(self, service):
assert isinstance(service, BaseService)
self.services.remove(service)
delattr(self.services, service.name)
def start(self):
for service in self.services.values():
service.start()
def main():
# config
import yaml
import io
import sys
import signal
import gevent
from peermanager import PeerManager
from jsonrpc import JSONRPCServer
from discovery import NodeDiscovery
import slogging
log = slogging.get_logger('app')
slogging.configure(config_string=':debug')
# read config
sample_config = """
p2p:
num_peers: 10
bootstrap_nodes:
# local bootstrap
# - enode://6ed2fecb28ff17dec8647f08aa4368b57790000e0e9b33a7b91f32c41b6ca9ba21600e9a8c44248ce63a71544388c6745fa291f88f8b81e109ba3da11f7b41b9@127.0.0.1:30303
# go_bootstrap
#- enode://6cdd090303f394a1cac34ecc9f7cda18127eafa2a3a06de39f6d920b0e583e062a7362097c7c65ee490a758b442acd5c80c6fce4b148c6a391e946b45131365b@54.169.166.226:30303
# cpp_bootstrap
- enode://4a44599974518ea5b0f14c31c4463692ac0329cb84851f3435e6d1b18ee4eae4aa495f846a0fa1219bd58035671881d44423876e57db2abd57254d0197da0ebe@5.1.83.226:30303
listen_host: 0.0.0.0
listen_port: 30303
node:
privkey_hex: 65462b0520ef7d3df61b9992ed3bea0c56ead753be7c8b3614e0ce01e4cac41b
"""
if len(sys.argv) == 1:
config = yaml.load(io.BytesIO(sample_config))
pubkey = crypto.privtopub(config['node']['privkey_hex'].decode('hex'))
config['node']['id'] = crypto.sha3(pubkey)
else:
fn = sys.argv[1]
log.info('loading config from', fn=fn)
config = yaml.load(open(fn))
# stop on every unhandled exception!
gevent.get_hub().SYSTEM_ERROR = BaseException # (KeyboardInterrupt, SystemExit, SystemError)
print config
# create app
app = BaseApp(config)
# register services
NodeDiscovery.register_with_app(app)
PeerManager.register_with_app(app)
# JSONRPCServer.register_with_app(app)
# start app
app.start()
# wait for interupt
evt = gevent.event.Event()
# gevent.signal(signal.SIGQUIT, gevent.kill) ## killall pattern
gevent.signal(signal.SIGQUIT, evt.set)
gevent.signal(signal.SIGTERM, evt.set)
gevent.signal(signal.SIGINT, evt.set)
evt.wait()
# finally stop
app.stop()
now = self.last_request = time.time()
gevent.sleep(self.ping_interval)
self.log.debug('latency', peer=self.proto, latency='%.3f' % self.latency())
if now - self.last_response > self.response_delay_threshold:
self.log.debug('unresponsive_peer', monitor=self)
self.proto.peer.report_error('not responding to ping')
self.proto.stop()
self.kill()
def stop(self):
self.log.debug('stopped', monitor=self)
self.kill()
########################################
log = slogging.get_logger('protocol.p2p')
class P2PProtocol(BaseProtocol):
"""
DEV P2P Wire Protocol
https://github.com/ethereum/wiki/wiki/%C3%90%CE%9EVp2p-Wire-Protocol
"""
protocol_id = 0
name = 'p2p'
version = 3
max_cmd_id = 15
def __init__(self, peer, service):
# required by P2PProtocol
import gevent
import socket
import atexit
import time
from gevent.server import StreamServer
from gevent.socket import create_connection, timeout
from service import WiredService
from protocol import BaseProtocol
from p2p_protocol import P2PProtocol
import kademlia
from peer import Peer
import crypto
import utils
import slogging
log = slogging.get_logger('p2p.peermgr')
class PeerManager(WiredService):
"""
todo:
connects new peers if there are too few
selects peers based on a DHT
keeps track of peer reputation
saves/loads peers (rather discovery buckets) to disc
connection strategy
for service which requires peers
while num peers > min_num_peers:
gen random id
resolve closest node address
from ethereum.specials import specials as default_specials
from config import Env, default_config
from db import BaseDB, EphemDB
from ethereum.exceptions import InvalidNonce, InsufficientStartGas, UnsignedTransaction, \
BlockGasLimitReached, InsufficientBalance, VerificationFailed, InvalidTransaction
import sys
if sys.version_info.major == 2:
from repoze.lru import lru_cache
else:
from functools import lru_cache
from slogging import get_logger
from timeit import default_timer as timer
null_address = b'\xff' * 20
log = get_logger('eth.block')
log_tx = get_logger('eth.pb.tx')
log_msg = get_logger('eth.pb.msg')
log_state = get_logger('eth.pb.msg.state')
# contract creating transactions send to an empty address
CREATE_CONTRACT_ADDRESS = b''
# DEV OPTIONS
SKIP_MEDSTATES = False
def rp(tx, what, actual, target):
return '%r: %r actual:%r target:%r' % (tx, what, actual, target)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# https://github.com/robnewton/JSON-RPC-Browser
import gevent
import gevent.wsgi
import gevent.queue
from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
from tinyrpc.transports.wsgi import WsgiServerTransport
from tinyrpc.server.gevent import RPCServerGreenlets
from tinyrpc.dispatch import RPCDispatcher
from service import BaseService
import slogging
log = slogging.get_logger('jsonrpc')
class JSONRPCServer(BaseService):
name = 'jsonrpc'
def __init__(self, app):
log.debug('initializing JSONRPCServer')
BaseService.__init__(self, app)
self.app = app
self.dispatcher = RPCDispatcher()
transport = WsgiServerTransport(queue_class=gevent.queue.Queue)
# start wsgi server as a background-greenlet
self.wsgi_server = gevent.wsgi.WSGIServer(('127.0.0.1', 5000), transport.handle)
self.rpc_server = RPCServerGreenlets(
