def __init__(self,
wallet,
socket_base,
ctx=None,
network_parameters=NetworkParameters(),
linger=500,
poll_timeout=200,
blocks: CilantroStorageDriver = None,
driver=BlockchainDriver()):
self.wallet = wallet
self.ctx = ctx or zmq.asyncio.Context()
self.address = network_parameters.resolve(socket_base,
ServiceType.BLOCK_SERVER,
bind=True)
super().__init__(socket_id=self.address,
wallet=self.wallet,
ctx=self.ctx,
linger=linger,
poll_timeout=poll_timeout)
self.blocks = blocks or CilantroStorageDriver(
key=self.wallet.signing_key())
self.driver = driver
self.log = get_logger('BlockServer')
def __init__(self,
ctx: zmq.asyncio.Context,
loop=asyncio.get_event_loop(),
domain='*',
cert_dir=CERT_DIR,
debug=True):
# Create the directory if it doesn't exist
self.cert_dir = pathlib.Path.home() / cert_dir
self.cert_dir.mkdir(parents=True, exist_ok=True)
self.ctx = ctx
self.domain = domain
self.loop = loop
self.log = get_logger('zmq.auth')
self.log.propagate = debug
# This should throw an exception if the socket already exist
try:
self.authenticator = AsyncioAuthenticator(context=self.ctx,
loop=self.loop)
self.authenticator.start()
self.authenticator.configure_curve(domain=self.domain,
location=self.cert_dir)
except ZMQBaseError:
pass
def __init__(self, driver=ContractDriver(), debug=True):
self.driver = driver
self.client = ContractingClient(driver=driver)
# All of this can be just derived from the blockchain driver without marking reads
# Should marks on default be false?
self.stamp_contract = self.client.get_contract('stamp_cost')
self.reward_contract = self.client.get_contract('rewards')
self.currency_contract = self.client.get_contract('currency')
self.election_house = self.client.get_contract('election_house')
self.foundation_contract = self.client.get_contract('foundation')
self.masternodes_contract = self.client.get_contract('masternodes')
self.delegates_contract = self.client.get_contract('delegates')
assert self.stamp_contract is not None, 'Stamp contract not in state.'
assert self.reward_contract is not None, 'Reward contract not in state.'
assert self.currency_contract is not None, 'Currency contract not in state.'
assert self.foundation_contract is not None, 'Foundation contract not in state.'
assert self.masternodes_contract is not None, 'Masternodes not in state.'
assert self.delegates_contract is not None, 'Delegates not in state.'
self.log = get_logger('RWM')
self.log.propagate = debug
self.dust_exponent = 8
def __init__(self, parallelism=4, *args, **kwargs):
super().__init__(*args, **kwargs)
# Number of core / processes we push to
self.parallelism = parallelism
self.executor = Executor(driver=self.driver)
self.work_inbox = WorkInbox(socket_id=self.network_parameters.resolve(
self.socket_base, ServiceType.INCOMING_WORK, bind=True),
driver=self.driver,
ctx=self.ctx,
client=self.client,
wallet=self.wallet)
self.pending_sbcs = set()
self.log = get_logger(f'DEL {self.wallet.vk_pretty[4:12]}')
self.masternode_socket_book = Peers(
wallet=self.wallet,
ctx=self.ctx,
parameters=self.parameters,
service_type=ServiceType.BLOCK_AGGREGATOR,
node_type=MN)
self.masternode_contract = self.client.get_contract('masternodes')
def __init__(self,
key,
distribute_writes=False,
config_path=cilantro_ee.__path__[0],
**kwargs):
self.state_id = ObjectId(OID)
self.log = get_logger("StorageDriver")
self.block_index_delta = defaultdict(dict)
self.send_req_blk_num = 0
super().__init__(key,
distribute_writes=distribute_writes,
config_path=config_path,
**kwargs)
def __init__(self,
total_contacts,
total_subblocks,
required_consensus=0.66,
acceptable_consensus=0.5):
self.total_contacts = total_contacts
self.total_subblocks = total_subblocks
self.required_consensus = required_consensus
# Acceptable consensus forces a block to complete. Anything below this will fail.
self.acceptable_consensus = acceptable_consensus
# Create an empty list to store the contenders as they come in
self.subblock_contenders = [None for _ in range(self.total_subblocks)]
self.log = get_logger('AGG')
def __init__(self,
socket_id,
ctx,
driver,
wallet,
expected_subblocks=4,
seconds_to_timeout=10):
self.expected_subblocks = expected_subblocks
self.sbc_inbox = SBCInbox(socket_id=socket_id,
ctx=ctx,
driver=driver,
expected_subblocks=self.expected_subblocks,
wallet=wallet)
self.driver = driver
self.seconds_to_timeout = seconds_to_timeout
self.log = get_logger('AGG')
def __init__(self,
contacts: VKBook,
driver: BlockchainDriver = BlockchainDriver(),
verify=True,
allow_current_block_num=False,
*args,
**kwargs):
self.q = []
self.contacts = contacts
self.driver = driver
self.verify = verify
self.quorum_ratio = 0.50
self.allow_current_block_num = allow_current_block_num
self.log = get_logger('NBN')
self.signers = len(
self.contacts.delegates
) # This has to be updated every block in case a delegate is added or removed
super().__init__(*args, **kwargs)
async def discover_nodes(ip_list,
pepper: bytes,
ctx: zmq.Context,
timeout=1000,
retries=10,
debug=True):
nodes_found = {}
one_found = False
retries_left = retries
log = get_logger('DiscoverNodes')
log.propagate = debug
log.info([str(ip) for ip in ip_list])
while not one_found and retries_left > 0:
tasks = [
ping(socket_id=ip, pepper=pepper, ctx=ctx, timeout=timeout)
for ip in ip_list
]
tasks = asyncio.gather(*tasks)
loop = asyncio.get_event_loop()
log.info('Sending pings to {} nodes.'.format(len(ip_list)))
if loop.is_running():
results = await asyncio.ensure_future(tasks)
else:
results = loop.run_until_complete(tasks)
for res in results:
ip, vk = res
if vk is not None:
nodes_found[str(ip)] = vk.hex()
log.info(f'Found {ip} with VK {vk}')
one_found = True
if not one_found:
retries_left -= 1
log.info(
'No one discovered... {} retried left.'.format(retries_left))
# Returns mapping of IP -> VK. VKs that return None are not stored in the dictionary.
return nodes_found
def __init__(self,
wallet: Wallet,
ctx: zmq.asyncio.Context,
blocks: CilantroStorageDriver = None,
state=BlockchainDriver(),
parameters: Parameters = None):
self.parameters = parameters
self.wallet = wallet
self.ctx = ctx
self.blocks = blocks
self.state = state
self.blocks_to_process = []
self.in_catchup = False
self.log = get_logger('Catchup')
def __init__(self,
socket_base: str,
ctx: zmq.asyncio.Context,
wallet,
contacts: VKBook,
network_parameters: NetworkParameters = NetworkParameters(),
debug=False):
self.socket_base = socket_base
self.ctx = ctx
self.wallet = wallet
self.network_parameters = network_parameters
self.contacts = contacts
self.peer_service_address = self.network_parameters.resolve(
socket_base, ServiceType.PEER)
self.sockets = {}
self.log = get_logger('Parameters')
self.log.propagate = debug
def __init__(self,
client: ContractingClient,
driver: BlockchainDriver = BlockchainDriver(),
verify=True,
debug=True,
*args,
**kwargs):
self.work = {}
self.driver = driver
self.verify = verify
self.client = client
self.masternode_contract = self.client.get_contract('masternodes')
self.todo = []
self.accepting_work = False
self.log = get_logger('DEL WI')
self.log.propagate = debug
super().__init__(*args, **kwargs)
async def ping(socket_id: cilantro_ee.sockets.struct.SocketStruct,
pepper: bytes,
ctx: zmq.Context,
timeout,
debug=False):
log = get_logger('Pinger')
log.propagate = debug
log.info(f'Pinging: {socket_id.zmq_url()}')
response = await services.get(socket_id=socket_id,
msg=b'',
ctx=ctx,
timeout=timeout)
log.info('Got response: {}'.format(response))
vk = None
if verify_vk_pepper(response, pepper):
log.info(
'Verifying key successfully extracted and message matches network pepper.'
)
vk, _ = unpack_pepper_msg(response)
return str(socket_id), vk
def __init__(self,
wallet,
params=NetworkParameters(),
ctx=zmq.asyncio.Context(),
bootnodes=[],
initial_mn_quorum=1,
initial_del_quorum=1,
mn_to_find=[],
del_to_find=[],
socket_base='tcp://0.0.0.0',
poll_timeout=200,
linger=1000,
debug=True,
mn_seed=None):
self.log = get_logger('NetworkService')
self.log.propagate = debug
self.mn_seed = mn_seed # Set this to a single masternode if you are joining the network!!
# General Instance Variables
self.wallet = wallet
self.ctx = ctx
self.bootnodes = bootnodes
self.ip = socket_base
# Peer Service Constants
self.params = params
self.socket_base = socket_base
self.peer_service_address = self.params.resolve(socket_base,
ServiceType.PEER,
bind=True)
self.event_server_address = self.params.resolve(socket_base,
ServiceType.EVENT,
bind=True)
self.peer_service = PeerServer(
self.peer_service_address,
event_address=self.event_server_address,
table={self.wallet.verifying_key().hex(): socket_base},
wallet=self.wallet,
ctx=self.ctx,
poll_timeout=poll_timeout,
linger=linger)
self.discovery_server_address = self.params.resolve(
self.socket_base, ServiceType.DISCOVERY, bind=True)
self.discovery_server = discovery.DiscoveryServer(
pepper=PEPPER.encode(),
socket_id=self.discovery_server_address,
wallet=self.wallet,
ctx=self.ctx,
poll_timeout=poll_timeout,
linger=linger)
# Quorum Constants
self.initial_mn_quorum = initial_mn_quorum
self.initial_del_quorum = initial_del_quorum
self.mn_to_find = mn_to_find
self.del_to_find = del_to_find
self.ready = False
self.outbox = services.Outbox(self.ctx)
from contracting.db.encoder import encode
from cilantro_ee.storage import MasterStorage, BlockchainDriver
from cilantro_ee.crypto.canonical import tx_hash_from_tx
from cilantro_ee.crypto.transaction import transaction_is_valid, \
TransactionNonceInvalid, TransactionProcessorInvalid, TransactionTooManyPendingException, \
TransactionSenderTooFewStamps, TransactionPOWProofInvalid, TransactionSignatureInvalid, TransactionStampsNegative
from cilantro_ee.messages.capnp_impl import capnp_struct as schemas
import os
import capnp
import ast
import ssl
import asyncio
log = get_logger("MN-WebServer")
transaction_capnp = capnp.load(
os.path.dirname(schemas.__file__) + '/transaction.capnp')
class ByteEncoder(_json.JSONEncoder):
def default(self, o, *args):
if isinstance(o, bytes):
return o.hex()
return super().default(self, o)
class WebServer:
def __init__(
self,
import aiohttp
import asyncio
import requests
import subprocess
import os, sys
from cilantro_ee.crypto.transaction import TransactionBuilder
from cilantro_ee.cli.utils import get_update_state, ask, validate_key
from cilantro_ee.cli.start import resolve_constitution
from cilantro_ee.logger.base import get_logger
log = get_logger('Cmd-upd')
async def cil_interface(server, packed_data, sleep=2):
async with aiohttp.ClientSession() as session:
r = await session.post(url=f'http://{server}:18080/', data=packed_data)
result = await r.json()
await asyncio.sleep(sleep)
return result
def trigger(pkg=None, iaddr=None):
my_wallet = validate_key(restart=False)
pepper = pkg #TODO replace with verified pepper pkg
kwargs = {
'pepper': pepper,
'initiator_vk': my_wallet.verifying_key().hex()
}
vk = my_wallet.verifying_key()
SERVER = f'http://{iaddr}:18080'
from cilantro_ee.logger.base import get_logger
import zmq
import asyncio
from zmq.utils import monitor
import pathlib
from zmq.auth.certs import load_certificate
from cilantro_ee.sockets.struct import SocketStruct
log = get_logger("BaseServices")
class Outbox:
def __init__(self, ctx: zmq.Context):
self.sockets = {}
self.ctx = ctx
def get_socket(self, socket_id, _type=zmq.DEALER, linger=500):
socket = self.sockets.get(str(socket_id))
# Connect and store if it doesn't exist
if socket is None:
socket = self.ctx.socket(_type)
socket.connect(str(socket_id))
socket.setsockopt(zmq.LINGER, linger)
self.sockets[str(socket_id)] = socket
return socket
async def get(self, socket_id, msg, timeout=1000):
socket = self.get_socket(socket_id, zmq.REQ)
import shlex
import sys
from getpass import getpass
import psutil
import pathlib
import subprocess
import ipaddress
import cilantro_ee
from checksumdir import dirhash
from contracting.client import ContractingClient
from cilantro_ee.storage.contract import BlockchainDriver
from cilantro_ee.logger.base import get_logger
from cilantro_ee.crypto.wallet import Wallet
log = get_logger('Cmd')
def validate_ip(address):
try:
ip = ipaddress.ip_address(address)
log.info('%s is a correct IP%s address.' % (ip, ip.version))
return ip
except ValueError:
log.error('address/netmask is invalid: %s' % address)
def build_pepper(pkg_dir_path=os.environ.get('CIL_PATH')):
if pkg_dir_path is None:
pkg_dir_path = '/Volumes/dev/lamden/cilantro-enterprise'
from contracting.stdlib.bridge.time import Datetime
from contracting.db.encoder import decode
from contracting.db.driver import encode_kv
from cilantro_ee.crypto.canonical import build_sbc_from_work_results, tx_hash_from_tx
from cilantro_ee.logger.base import get_logger
import os
import capnp
from datetime import datetime
import hashlib
import heapq
import cilantro_ee.messages.capnp_impl.capnp_struct as schemas
transaction_capnp = capnp.load(
os.path.dirname(schemas.__file__) + '/transaction.capnp')
log = get_logger('EXE')
def execute_tx(executor: Executor,
transaction,
stamp_cost,
environment: dict = {},
debug=True):
# Deserialize Kwargs. Kwargs should be serialized JSON moving into the future for DX.
kwargs = decode(transaction.payload.kwargs)
output = executor.execute(sender=transaction.payload.sender.hex(),
contract_name=transaction.payload.contractName,
function_name=transaction.payload.functionName,
stamps=transaction.payload.stampsSupplied,
stamp_cost=stamp_cost,
from contracting.db.driver import ContractDriver, Driver
from contracting.db.encoder import decode
from cilantro_ee.logger.base import get_logger
BLOCK_HASH_KEY = '_current_block_hash'
BLOCK_NUM_KEY = '_current_block_num'
NONCE_KEY = '__n'
PENDING_NONCE_KEY = '__pn'
log = get_logger('STATE')
class BlockchainDriver(ContractDriver):
def get_latest_block_hash(self):
block_hash = self.driver.get(BLOCK_HASH_KEY)
if block_hash is None:
return '0' * 64
return block_hash
def set_latest_block_hash(self, v: str):
if type(v) == bytes:
v = v.hex()
assert len(v) == 64, 'Hash provided is not 32 bytes.'
self.driver.set(BLOCK_HASH_KEY, v)
latest_block_hash = property(get_latest_block_hash, set_latest_block_hash)
def get_latest_block_num(self) -> int:
num = self.driver.get(BLOCK_NUM_KEY)
if num is None:
def __init__(self,
socket_base,
ctx: zmq.asyncio.Context,
wallet,
constitution: dict,
overwrite=False,
bootnodes=[],
network_parameters=NetworkParameters(),
driver=BlockchainDriver(),
mn_seed=None,
debug=True,
store=False):
self.driver = driver
self.store = store
self.blocks = None
if self.store:
self.blocks = MasterStorage()
self.waiting_for_confirmation = False
self.log = get_logger('NODE')
self.log.propagate = debug
self.log.info(constitution)
self.socket_base = socket_base
self.wallet = wallet
self.ctx = ctx
self.ctx.max_sockets = 50_000
self.client = ContractingClient(
driver=self.driver,
submission_filename=cilantro_ee.contracts.__path__[0] +
'/submission.s.py')
# Sync contracts
sync.submit_from_genesis_json_file(cilantro_ee.contracts.__path__[0] +
'/genesis.json',
client=self.client)
sync.submit_node_election_contracts(
initial_masternodes=constitution['masternodes'],
boot_mns=constitution['masternode_min_quorum'],
initial_delegates=constitution['delegates'],
boot_dels=constitution['delegate_min_quorum'],
client=self.client)
self.driver.commit()
self.driver.clear_pending_state()
self.contacts = VKBook(
client=self.client,
boot_mn=constitution['masternode_min_quorum'],
boot_del=constitution['delegate_min_quorum'],
)
self.current_masters = deepcopy(self.contacts.masternodes)
self.current_delegates = deepcopy(self.contacts.delegates)
self.parameters = Parameters(socket_base,
ctx,
wallet,
contacts=self.contacts)
self.socket_authenticator = SocketAuthenticator(ctx=self.ctx)
self.elect_masternodes = self.client.get_contract('elect_masternodes')
self.elect_delegates = self.client.get_contract('elect_delegates')
self.masternode_contract = self.client.get_contract('masternodes')
self.delegate_contract = self.client.get_contract('delegates')
self.update_sockets()
# Cilantro version / upgrade
self.version_state = self.client.get_contract('upgrade')
self.active_upgrade = self.version_state.quick_read('upg_lock')
self.tol_mn = self.version_state.quick_read('tol_mn')
self.tot_dl = self.version_state.quick_read('tot_dl')
if self.tol_mn is None:
self.tol_mn = 0
if self.tot_dl is None:
self.tot_dl = 0
self.all_votes = self.tol_mn + self.tot_dl
self.mn_votes = self.version_state.quick_read('mn_vote')
self.dl_votes = self.version_state.quick_read('dl_vote')
# self.pending_cnt = self.all_votes - self.vote_cnt
# stuff
self.network_parameters = network_parameters
self.bootnodes = bootnodes
self.constitution = constitution
self.overwrite = overwrite
# Should have a function to get the current NBN
self.block_fetcher = BlockFetcher(
wallet=self.wallet,
ctx=self.ctx,
parameters=self.parameters,
)
self.network = Network(
wallet=self.wallet,
ctx=self.ctx,
socket_base=socket_base,
bootnodes=self.bootnodes,
params=self.network_parameters,
initial_del_quorum=deepcopy(self.contacts.delegate_quorum_min),
initial_mn_quorum=deepcopy(self.contacts.masternode_quorum_min),
mn_to_find=deepcopy(self.contacts.masternodes),
del_to_find=deepcopy(self.contacts.delegates),
mn_seed=mn_seed)
self.nbn_inbox = NBNInbox(socket_id=self.network_parameters.resolve(
self.socket_base,
service_type=ServiceType.BLOCK_NOTIFICATIONS,
bind=True),
ctx=self.ctx,
driver=self.driver,
contacts=self.contacts,
wallet=wallet)
self.reward_manager = RewardManager(driver=self.driver, debug=True)
self.running = False
import zmq.asyncio
import cilantro_ee.sockets.reqrep
import cilantro_ee.sockets.struct
from cilantro_ee.sockets.inbox import SecureAsyncInbox, AsyncInbox
from cilantro_ee.logger.base import get_logger
from cilantro_ee.crypto.wallet import Wallet, _verify
from cilantro_ee.sockets import services
import asyncio
log = get_logger('DiscoveryService')
'''
DiscoverServer
Returns a message of the signed pepper and VK
'''
TIMEOUT = 1000
LINGER = 500
POLL = 50
class DiscoveryServer(AsyncInbox):
def __init__(self, pepper: bytes, *args, **kwargs):
super().__init__(*args, **kwargs)
self.pepper = pepper
self.response = self.wallet.verifying_key() + self.wallet.sign(
self.pepper)
async def handle_msg(self, _id, msg):
await self.return_msg(_id, self.response)
from cilantro_ee.crypto.wallet import Wallet
from cilantro_ee.messages.message import Message
from cilantro_ee.messages.message_type import MessageType
import hashlib
import time
from cilantro_ee.logger.base import get_logger
log = get_logger('TXBATCHER')
class TransactionBatcher:
def __init__(self, wallet: Wallet, queue):
self.wallet = wallet
self.queue = queue
def make_empty_batch(self):
timestamp = time.time()
h = hashlib.sha3_256()
h.update('{}'.format(timestamp).encode())
input_hash = h.digest()
signature = self.wallet.sign(input_hash)
msg = Message.get_signed_message_packed_2(
wallet=self.wallet,
msg_type=MessageType.TRANSACTION_BATCH,
transactions=[],
timestamp=timestamp,
import os
import capnp
from cilantro_ee.messages.capnp_impl import capnp_struct as schemas
import bson
import hashlib
from cilantro_ee.crypto.merkle_tree import merklize
from cilantro_ee.messages import Message, MessageType
from copy import deepcopy
from cilantro_ee.logger.base import get_logger
log = get_logger('CANON')
subblock_capnp = capnp.load(os.path.dirname(schemas.__file__) + '/subblock.capnp')
block_capnp = capnp.load(os.path.dirname(schemas.__file__) + '/blockdata.capnp')
GENESIS_HASH = b'\x00' * 32
def format_dictionary(d: dict) -> dict:
for k, v in d.items():
assert type(k) == str, 'Non-string key types not allowed.'
if type(v) == list:
for i in range(len(v)):
if isinstance(v[i], dict):
v[i] = format_dictionary(v[i])
elif isinstance(v, dict):
d[k] = format_dictionary(v)
return {k: v for k, v in sorted(d.items())}
