def _initialize_background(self):
self._status = 'STARTING'
if self._gateway_config:
self._w3 = make_w3(self._gateway_config)
else:
from web3.auto.infura import w3
self._w3 = w3
self._ens = ENS.fromWeb3(self._w3)
if self._name_category in ['ens', 'reverse_ens']:
if self._name_category == 'reverse_ens':
name = ''.join(reversed(self._name_or_address.split('.')))
else:
name = self._name_or_address
self._address = self._ens.address(name)
elif self._name_category == 'eth':
self._address = self._w3.toChecksumAddress(self._name_or_address)
else:
assert False, 'should not arrive here'
# https://web3py.readthedocs.io/en/stable/contracts.html#web3.contract.Contract
# https://web3py.readthedocs.io/en/stable/web3.eth.html#web3.eth.Eth.contract
# self._contract = self._w3.eth.contract(address=self.CONTRACT_ADDRESS, abi=self.CONTRACT_ABI)
# FIXME: get IPFS hash, download file, unzip seeders
self._status = 'RUNNING'
async def _do_market_realm(self, details):
self._w3 = make_w3(_INFURA_CONFIG)
xbr.setProvider(self._w3)
command = self.config.extra['command']
assert command in ['open-channel', 'get-channel']
if command == 'get-channel':
market_oid = self.config.extra['market']
delegate = self.config.extra['delegate']
channel_type = self.config.extra['channel_type']
if channel_type == ChannelType.PAYMENT:
await self._do_get_active_payment_channel(market_oid, delegate)
elif channel_type == ChannelType.PAYING:
await self._do_get_active_paying_channel(market_oid, delegate)
else:
assert False, 'should not arrive here'
elif command == 'open-channel':
market_oid = self.config.extra['market']
channel_oid = self.config.extra['channel']
channel_type = self.config.extra['channel_type']
delegate = self.config.extra['delegate']
amount = self.config.extra['amount']
await self._do_open_channel(market_oid, channel_oid, channel_type, delegate, amount)
else:
assert False, 'should not arrive here'
def test_ens_resolve_names(self):
from autobahn.xbr import make_w3
from ens.main import ENS
w3 = make_w3(self.gw_config)
ens = ENS.fromWeb3(w3)
for name, adr in [
('wamp-proto.eth', '0x66267d0b1114cFae80C37942177a846d666b114a'),
]:
_adr = ens.address(name)
self.assertEqual(adr, _adr)
def __init__(self, config=None, reactor=None, personality=None):
# WorkerController derives of NativeProcess, which will set self._reactor
WorkerController.__init__(self, config=config, reactor=reactor, personality=personality)
worker_options_extra = dict(config.extra.extra)
self._database_config = worker_options_extra['database']
self._blockchain_config = worker_options_extra['blockchain']
self._ipfs_files_directory = worker_options_extra.get('ipfs_files_directory', './.ipfs_files')
# xbrmm worker status
self._status = None
# map of market makers by ID
self._makers = {}
self._maker_adr2id = {}
# open xbrmm worker database, containing a replicate of xbr on-chain data (other than
# channels, which are market specific and stored in the market maker database of the maker of that market)
self._dbpath = os.path.abspath(
self._database_config.get('dbpath', './.xbrmm-{}-db'.format(config.extra.worker)))
self._db = zlmdb.Database(dbpath=self._dbpath,
maxsize=self._database_config.get('maxsize', 2**30),
readonly=False,
sync=True)
self._db.__enter__()
# generic database object metadata
self._meta = cfxdb.meta.Schema.attach(self._db)
# xbr database schema
self._xbr = cfxdb.xbr.Schema.attach(self._db)
# xbr market maker schema
self._xbrmm = cfxdb.xbrmm.Schema.attach(self._db)
# event object too coordinate exit of blockchain monitor background check
self._run_monitor = None
# blockchain gateway configuration
self._bc_gw_config = self._blockchain_config['gateway']
self.log.info('Initializing Web3 from blockchain gateway configuration\n\n{gateway}\n',
gateway=pformat(self._bc_gw_config))
self._w3 = make_w3(self._bc_gw_config)
xbr.setProvider(self._w3)
self._chain_id = self._blockchain_config.get('chain_id', 1)
self.log.info('Using chain ID {chain_id}', chain_id=hlid(self._chain_id))
# To be initiated once cbdir variable gets available
self._ipfs_files_dir = os.path.join(config.extra.cbdir, self._ipfs_files_directory)
async def _do_market_realm(self, details):
profile = self.config.extra['profile']
blockchain_gateway = {
"type": "infura",
"network": profile.infura_network,
"key": profile.infura_key,
"secret": profile.infura_secret
}
self._w3 = make_w3(blockchain_gateway)
xbr.setProvider(self._w3)
command = self.config.extra['command']
assert command in ['open-channel', 'get-channel']
if command == 'get-channel':
market_oid = self.config.extra['market']
delegate = self.config.extra['delegate']
channel_type = self.config.extra['channel_type']
if channel_type == ChannelType.PAYMENT:
await self._do_get_active_payment_channel(market_oid, delegate)
elif channel_type == ChannelType.PAYING:
await self._do_get_active_paying_channel(market_oid, delegate)
else:
assert False, 'should not arrive here'
elif command == 'open-channel':
# market in which to open the new buyer/seller (payment/paying) channel
market_oid = self.config.extra['market']
# read UUID of the new channel to be created from command line OR auto-generate a new one
channel_oid = self.config.extra['channel'] or uuid.uuid4()
# buyer/seller (payment/paying) channel
channel_type = self.config.extra['channel_type']
# the delgate allowed to use the channel
delegate = self.config.extra['delegate']
# amount of market coins for initial channel balance
amount = self.config.extra['amount']
# now open the channel ..
await self._do_open_channel(market_oid, channel_oid, channel_type, delegate, amount)
else:
assert False, 'should not arrive here'
def load_config(self, configfile=None):
"""
Load the node configuration of CFC itself. The base node configuration
is built into CFC, but can be overridden (partially) and extended
by providing a regular, local node configuration file.
When such a file exists:
- and it contains a controller section, these take precedence over
the builtin values.
- and it contains a workers section (a list of workers), these are
_added_ to the builtin workers.
"""
# load builtin node configuration as default
#
filename = pkg_resources.resource_filename('crossbar',
self.DEFAULT_CONFIG_PATH)
with open(filename) as f:
config = json.load(f)
config_path = filename
config_source = self.CONFIG_SOURCE_DEFAULT
# self.personality.check_config(self.personality, config)
self.log.debug('Built-in node configuration loaded and checked')
# try loading node configuration from blockchain (overwriting the previously loaded default config)
# FIXME: domain/node configuration was removed from the XBRNetwork contract (for now)
# see: https://github.com/crossbario/xbr-protocol/pull/36
if False:
gateway_config = {
"type": "user",
"http": "http://localhost:1545",
}
self._w3 = make_w3(gateway_config)
xbr.setProvider(self._w3)
if self._w3.isConnected():
self.log.info(
'Connected to Ethereum network {network} at gateway "{gateway_url}"',
network=self._w3.version.network,
gateway_url=gateway_config['http'])
if 'XBR_DEBUG_TOKEN_ADDR' in os.environ:
self.log.info(
'XBR Token contract address {token_addr} set from environment',
token_addr=hlid(os.environ['XBR_DEBUG_TOKEN_ADDR']))
if 'XBR_DEBUG_NETWORK_ADDR' in os.environ:
self.log.info(
'XBR Network contract address {network_addr} set from environment',
network_addr=hlid(
os.environ['XBR_DEBUG_NETWORK_ADDR']))
xbr_node_id = xbr.xbrnetwork.functions.getNodeByKey(
self.key.public_key()).call()
if xbr_node_id != b'\x00' * 16:
assert (len(xbr_node_id) == 16)
xbr_node_domain = xbr.xbrnetwork.functions.getNodeDomain(
xbr_node_id).call()
xbr_node_type = xbr.xbrnetwork.functions.getNodeType(
xbr_node_id).call()
xbr_node_config = xbr.xbrnetwork.functions.getNodeConfig(
xbr_node_id).call()
# FIXME: for testing use hard-coded "trial license" (no. 20010)
xbr_node_license = 20010
# xbr_node_license = xbr.xbrnetwork.functions.getNodeLicense(xbr_node_id).call()
self.log.info(
'Node is configured for XBR (xbr_node_id="{xbr_node_id}", xbr_node_domain="{xbr_node_domain}", xbr_node_type="{xbr_node_type}", xbr_node_license="{xbr_node_license}", xbr_node_config="{xbr_node_config}")',
xbr_node_id=hlid(
'0x' + binascii.b2a_hex(xbr_node_id).decode()),
xbr_node_domain=hlid(
'0x' + binascii.b2a_hex(xbr_node_domain).decode()),
xbr_node_type=hlid(xbr_node_type),
xbr_node_config=hlid(xbr_node_config),
xbr_node_license=hlid(xbr_node_license))
self._license = License(xbr_node_license)
self.log.info(
'Node is registered in the XBR network with license type={license}',
license=self._license.type)
if xbr_node_config:
if 'IPFS_GATEWAY_URL' in os.environ:
ipfs_gateway_url = os.environ['IPFS_GATEWAY_URL']
self.log.info(
'Using explicit IPFS gateway URL {ipfs_gateway_url} from environment variable IPFS_GATEWAY_URL',
ipfs_gateway_url=hlid(ipfs_gateway_url))
else:
ipfs_gateway_url = 'https://ipfs.infura.io:5001'
self.log.info(
'Using default IPFS Infura gateway URL {ipfs_gateway_url}',
ipfs_gateway_url=hlid(ipfs_gateway_url))
ipfs_config_url = '{}/api/v0/cat?arg={}&encoding=json'.format(
ipfs_gateway_url, xbr_node_config)
resp = requests.get(ipfs_config_url)
xbr_node_config_data = resp.json()
from pprint import pprint
pprint(xbr_node_config_data)
self.personality.check_config(self.personality,
xbr_node_config_data)
if 'controller' not in xbr_node_config_data:
xbr_node_config_data['controller'] = {}
if 'id' not in xbr_node_config_data['controller']:
xbr_node_config_data['controller']['id'] = str(
uuid.UUID(bytes=xbr_node_id))
self.log.info(
'Deriving node ID {node_id} from XBR network node ID',
node_id=hlid(
xbr_node_config_data['controller']['id']))
else:
self.log.info(
'Setting node ID {node_id} from XBR network node configuration',
node_id=hlid(
xbr_node_config_data['controller']['id']))
self._config = xbr_node_config_data
self.log.info(
'Node configuration loaded from XBR network (xbr_node_id="{xbr_node_id}")',
xbr_node_id=hlid(
'0x' + binascii.b2a_hex(xbr_node_id).decode()))
else:
self.log.debug(
'There is no node configuration stored in XBR network (xbr_node_id="{xbr_node_id}")',
xbr_node_id=hlid(
'0x' + binascii.b2a_hex(xbr_node_id).decode()))
config_source = self.CONFIG_SOURCE_XBRNETWORK
else:
self.log.warn(
'Could not find node public key on blockchain yet')
else:
self.log.warn('Could not connect to Ethereum blockchain')
# allow extending/overriding the node configuration loaded with local config files
#
if configfile:
config_source = self.CONFIG_SOURCE_LOCALFILE
config_path = os.path.abspath(os.path.join(self._cbdir,
configfile))
self.log.info(
'Expanding built-in node configuration from local file {configpath}',
configpath=hlid(config_path))
with open(config_path) as f:
custom_config = json.load(f)
# as an overriding config file does not need to be a fully valid config in itself, do _not_ check it!
# self.personality.check_config(self.personality, custom_config)
config = merge_config(config, custom_config)
# check the final, assembled initial node configuration to apply
#
self.personality.check_config(self.personality, config)
self._config = config
self.log.debug(
'Master node config after (effective after merge):\n{config}',
config=pformat(config))
return config_source, config_path
def test_connect_w3_autobahn(self):
from autobahn.xbr import make_w3
w3 = make_w3(self.gw_config)
self.assertTrue(w3.isConnected())
def _monitor_blockchain(self, gateway_config, scan_from_block, period=300):
"""
:param gateway_config:
:param scan_from_block:
:param period:
:return:
"""
w3 = make_w3(gateway_config)
initial_delay = 2
self.log.info(
'Start monitoring of blockchain ({blockchain_type}) on thread {thread_id} in {initial_delay} seconds, iterating every {period} seconds ..',
blockchain_type=str(self._bc_gw_config['type']),
initial_delay=hlval(initial_delay),
period=hlval(period),
thread_id=hlval(int(threading.get_ident())))
# using normal blocking call here, as we are running on a background thread!
time.sleep(initial_delay)
def _process_Token_Transfer(transactionHash, blockHash, args):
# event Transfer(address indexed from, address indexed to, uint256 value);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - {value} XBR token transferred (on-chain) from {_from} to {_to})',
event=hlcontract('XBRToken.Transfer'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
value=hlval(int(args.value / 10**18)),
_from=hlval(args['from']),
_to=hlval(args.to))
stored = False
with self._db.begin(write=True) as txn:
transactionHash = bytes(transactionHash)
token_transfer = self._xbr.token_transfers[txn, transactionHash]
if token_transfer:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('TokenTransfer'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
token_transfer = cfxdb.xbr.token.TokenTransfer()
token_transfer.tx_hash = transactionHash
token_transfer.block_hash = bytes(blockHash)
token_transfer.from_address = bytes(HexBytes(args['from']))
token_transfer.to_address = bytes(HexBytes(args.to))
token_transfer.value = args.value
self._xbr.token_transfers[txn, transactionHash] = token_transfer
stored = True
if stored:
self.log.info('new {contract}(tx_hash={tx_hash}) record stored database!',
contract=hlcontract('TokenTransfer'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
def _process_Token_Approval(transactionHash, blockHash, args):
# event Approval(address indexed from, address indexed to, uint256 value);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - {value} XBR token approved (on-chain) from owner {owner} to spender {spender})',
event=hlcontract('XBRToken.Approval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
value=hlval(int(args.value / 10**18)),
owner=hlval(args.owner),
spender=hlval(args.spender))
stored = False
with self._db.begin(write=True) as txn:
transactionHash = bytes(transactionHash)
token_approval = self._xbr.token_approvals[txn, transactionHash]
if token_approval:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('TokenApproval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
token_approval = cfxdb.xbr.token.TokenApproval()
token_approval.tx_hash = transactionHash
token_approval.block_hash = bytes(blockHash)
token_approval.owner_address = bytes(HexBytes(args.owner))
token_approval.spender_address = bytes(HexBytes(args.spender))
token_approval.value = args.value
self._xbr.token_approvals[txn, transactionHash] = token_approval
stored = True
if stored:
self.log.info('new {contract}(tx_hash={tx_hash}) record stored database!',
contract=hlcontract('TokenApproval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
def _process_Network_MemberRegistered(transactionHash, blockHash, args):
# /// Event emitted when a new member joined the XBR Network.
# event MemberCreated (address indexed member, string eula, string profile, MemberLevel level);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - XBR member created at address {address})',
event=hlcontract('XBRNetwork.MemberCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
address=hlid(args.member))
member_adr = bytes(HexBytes(args.member))
if args.eula:
h = multihash.decode(multihash.from_b58_string(args.eula))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRNetwork.MemberCreated - eula "{eula}" is not an IPFS (sha2-256) b58-encoded multihash',
eula=hlval(args.eula))
if args.profile:
h = multihash.decode(multihash.from_b58_string(args.profile))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRNetwork.MemberCreated - profile "{profile}" is not an IPFS (sha2-256) b58-encoded multihash',
eula=hlval(args.profile))
stored = False
with self._db.begin(write=True) as txn:
member = self._xbr.members[txn, member_adr]
if member:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('TokenApproval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
member = cfxdb.xbr.member.Member()
member.address = member_adr
member.timestamp = np.datetime64(time_ns(), 'ns')
member.registered = args.registered
member.eula = args.eula
member.profile = args.profile
member.level = args.level
self._xbr.members[txn, member_adr] = member
stored = True
if stored:
self.log.info('new {contract}(member_adr={member_adr}) record stored database!',
contract=hlcontract('MemberCreated'),
member_adr=hlid('0x' + binascii.b2a_hex(member_adr).decode()))
def _process_Network_MemberRetired(transactionHash, blockHash, args):
# /// Event emitted when a member leaves the XBR Network.
# event MemberRetired (address member);
self.log.warn('_process_Network_MemberRetired not implemented')
def _process_Market_MarketCreated(transactionHash, blockHash, args):
# /// Event emitted when a new market was created.
# event MarketCreated (bytes16 indexed marketId, uint32 marketSeq, address owner, string terms, string meta,
# address maker, uint256 providerSecurity, uint256 consumerSecurity, uint256 marketFee);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - XBR market created with ID {market_id})',
event=hlcontract('XBRMarket.MarketCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
market_id=hlid(uuid.UUID(bytes=args.marketId)))
market_id = uuid.UUID(bytes=args.marketId)
if args.terms:
h = multihash.decode(multihash.from_b58_string(args.terms))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRMarket.MarketCreated - terms "{terms}" is not an IPFS (sha2-256) b58-encoded multihash',
terms=hlval(args.terms))
if args.meta:
h = multihash.decode(multihash.from_b58_string(args.meta))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRMarket.MarketCreated - meta "{meta}" is not an IPFS (sha2-256) b58-encoded multihash',
meta=hlval(args.meta))
stored = False
with self._db.begin(write=True) as txn:
market = self._xbr.markets[txn, market_id]
if market:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('MarketCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
market = cfxdb.xbr.market.Market()
market.market = market_id
market.timestamp = np.datetime64(time_ns(), 'ns')
# FIXME
# market.created = args.created
market.seq = args.marketSeq
market.owner = bytes(HexBytes(args.owner))
market.terms = args.terms
market.meta = args.meta
market.maker = bytes(HexBytes(args.maker))
market.provider_security = args.providerSecurity
market.consumer_security = args.consumerSecurity
market.market_fee = args.marketFee
self._xbr.markets[txn, market_id] = market
stored = True
if stored:
self.log.info('new {contract}(market_id={market_id}) record stored database!',
contract=hlcontract('MarketCreated'),
market_id=hlid(market_id))
def _process_Market_MarketUpdated(transactionHash, blockHash, args):
# /// Event emitted when a market was updated.
# event MarketUpdated (bytes16 indexed marketId, uint32 marketSeq, address owner, string terms, string meta,
# address maker, uint256 providerSecurity, uint256 consumerSecurity, uint256 marketFee);
self.log.warn('_process_Market_MarketUpdated not implemented')
def _process_Market_MarketClosed(transactionHash, blockHash, args):
# /// Event emitted when a market was closed.
# event MarketClosed (bytes16 indexed marketId);
self.log.warn('_process_Market_MarketClosed not implemented')
def _process_Market_ActorJoined(transactionHash, blockHash, args):
# Event emitted when a new actor joined a market.
# event ActorJoined (bytes16 indexed marketId, address actor, uint8 actorType, uint joined, uint256 security, string meta);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - XBR market actor {actor} joined market {market_id})',
event=hlcontract('XBRMarket.ActorJoined'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
actor=hlid(args.actor),
market_id=hlid(uuid.UUID(bytes=args.marketId)))
market_id = uuid.UUID(bytes=args.marketId)
actor_adr = bytes(HexBytes(args.actor))
actor_type = int(args.actorType)
if args.meta:
h = multihash.decode(multihash.from_b58_string(args.meta))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRMarket.MarketCreated - meta "{meta}" is not an IPFS (sha2-256) b58-encoded multihash',
terms=hlval(args.meta))
stored = False
with self._db.begin(write=True) as txn:
actor = self._xbr.actors[txn, (market_id, actor_adr, actor_type)]
if actor:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('MarketCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
actor = cfxdb.xbr.actor.Actor()
actor.timestamp = np.datetime64(time_ns(), 'ns')
actor.market = market_id
actor.actor = actor_adr
actor.actor_type = actor_type
actor.joined = args.joined
actor.security = args.security
actor.meta = args.meta
self._xbr.actors[txn, (market_id, actor_adr, actor_type)] = actor
stored = True
if stored:
self.log.info(
'new {contract}(market_id={market_id}, actor_adr={actor_adr}, actor_type={actor_type}) record stored database!',
contract=hlcontract('ActorJoined'),
market_id=hlid(market_id),
actor_adr=hlid('0x' + binascii.b2a_hex(actor_adr).decode()),
actor_type=hlid(actor_type))
def _process_Market_ActorLeft(transactionHash, blockHash, args):
self.log.warn('_process_Market_ActorLeft not implemented')
def _process_Market_ConsentSet(transactionHash, blockHash, args):
# Event emitted when a consent is set
# emit ConsentSet(member, updated, marketId, delegate, delegateType,
# apiCatalog, consent, servicePrefix);
self.log.info('{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) ..',
event=hlcontract('XBRMarket.ConsentSet'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()))
catalog_oid = uuid.UUID(bytes=args.apiCatalog)
member = uuid.UUID(bytes=args.member)
delegate = args.delegate
delegate_type = args.delegateType
market_oid = uuid.UUID(bytes=args.marketId)
with self._db.begin(write=True) as txn:
consent = self._xbr.consents[txn, (catalog_oid, member, delegate, delegate_type, market_oid)]
consent.synced = True
def _process_Catalog_CatalogCreated(transactionHash, blockHash, args):
# Event emitted when a new API catalog is created
# emit CatalogCreated(catalogId, created, catalogSeq, owner, terms, meta);
self.log.info('{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) ..',
event=hlcontract('XBRCatalog.CatalogCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()))
catalog_oid = uuid.UUID(bytes=args.catalogId)
owner = bytes(HexBytes(args.owner))
created = np.datetime64(time_ns(), 'ns')
with self._db.begin(write=True) as txn:
catalog = cfxdb.xbr.catalog.Catalog()
catalog.oid = catalog_oid
catalog.timestamp = created
catalog.seq = args.catalogSeq
catalog.owner = owner
catalog.terms = args.terms
catalog.meta = args.meta
self._xbr.catalogs[txn, catalog_oid] = catalog
deferToThread(self._download_ipfs_file, args.meta)
def _process_Catalog_ApiPublished(transactionHash, blockHash, args):
self.log.warn('_process_Catalog_ApiPublished not implemented')
def _process_Channel_Opened(transactionHash, blockHash, args):
# Event emitted when a new XBR data market has opened.
# event Opened(XBRTypes.ChannelType ctype, bytes16 indexed marketId, bytes16 indexed channelId,
# address actor, address delegate, address marketmaker, address recipient,
# uint256 amount, bytes signature);
self.log.info('{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) ..',
event=hlcontract('XBRChannel.Opened'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()))
channel_oid = uuid.UUID(bytes=args.channelId)
marketmaker_adr = bytes(HexBytes(args.marketmaker))
marketmaker_adr_str = web3.Web3.toChecksumAddress(marketmaker_adr)
# we only persist data for xbr markets operated by one of the market makers we run in this worker
if marketmaker_adr_str not in self._maker_adr2id or self._maker_adr2id[
marketmaker_adr_str] not in self._makers:
self.log.info(
'{event}: skipping channel (channel {channel_oid} in market with market maker address {marketmaker_adr} is in for any market in this markets worker)',
event=hlcontract('XBRChannel.Opened'),
channel_oid=hlid(channel_oid),
marketmaker_adr=hlid(marketmaker_adr_str))
return
# prepare new channel data
channel_type = int(args.ctype)
market_oid = uuid.UUID(bytes=args.marketId)
actor_adr = bytes(HexBytes(args.actor))
delegate_adr = bytes(HexBytes(args.delegate))
recipient_adr = bytes(HexBytes(args.recipient))
amount = int(args.amount)
# FIXME
# signature = bytes(HexBytes(args.signature))
# FIXME
# member_oid = uuid.UUID()
# get the market maker by address
maker = self._makers[self._maker_adr2id[marketmaker_adr_str]]
# the market maker specific embedded database and schema
db = maker.db
xbrmm = maker.schema
# depending on channel type, different database schema classes and tables are used
if channel_type == cfxdb.xbrmm.ChannelType.PAYMENT:
channel = cfxdb.xbrmm.PaymentChannel()
channels = xbrmm.payment_channels
channels_by_delegate = xbrmm.idx_payment_channel_by_delegate
balance = cfxdb.xbrmm.PaymentChannelBalance()
balances = xbrmm.payment_balances
elif channel_type == cfxdb.xbrmm.ChannelType.PAYING:
channel = cfxdb.xbrmm.PayingChannel()
channels = xbrmm.paying_channels
channels_by_delegate = xbrmm.idx_paying_channel_by_delegate
balance = cfxdb.xbrmm.PayingChannelBalance()
balances = xbrmm.paying_balances
else:
assert False, 'should not arrive here'
# fill in information for newly replicated channel
channel.market_oid = market_oid
# FIXME
# channel.member_oid = member_oid
channel.channel_oid = channel_oid
channel.timestamp = np.datetime64(time_ns(), 'ns')
# channel.open_at = None
# FIXME: should read that from even args after deployment of
# https://github.com/crossbario/xbr-protocol/pull/138
channel.seq = 1
channel.channel_type = channel_type
channel.marketmaker = marketmaker_adr
channel.actor = actor_adr
channel.delegate = delegate_adr
channel.recipient = recipient_adr
channel.amount = amount
# FIXME
channel.timeout = 0
channel.state = cfxdb.xbrmm.ChannelState.OPEN
# FIXME
# channel.open_sig = signature
# create an off-chain balance record for the channel with remaining == initial amount
balance.remaining = channel.amount
# FIXME: should read that from even args after deployment of
# https://github.com/crossbario/xbr-protocol/pull/138
balance.seq = 1
# now store the new channel and balance in the database
stored = False
cnt_channels_before = 0
cnt_channels_by_delegate_before = 0
cnt_channels_after = 0
cnt_channels_by_delegate_after = 0
with db.begin(write=True) as txn:
if channels[txn, channel_oid]:
self.log.warn('{event}: channel already stored in database [channel_oid={channel_oid}]',
event=hlcontract('XBRChannel.Opened'),
channel_oid=hlid(channel_oid))
else:
cnt_channels_before = channels.count(txn)
cnt_channels_by_delegate_before = channels_by_delegate.count(txn)
# store the channel along with the off-chain balance
channels[txn, channel_oid] = channel
balances[txn, channel_oid] = balance
stored = True
cnt_channels_after = channels.count(txn)
cnt_channels_by_delegate_after = channels_by_delegate.count(txn)
self.log.info(
'{event} DB result: stored={stored}, cnt_channels_before={cnt_channels_before}, cnt_channels_by_delegate_before={cnt_channels_by_delegate_before}, cnt_channels_after={cnt_channels_after}, cnt_channels_by_delegate_after={cnt_channels_by_delegate_after}',
event=hlcontract('XBRChannel.Opened'),
stored=hlval(stored),
cnt_channels_before=hlval(cnt_channels_before),
cnt_channels_by_delegate_before=hlval(cnt_channels_by_delegate_before),
cnt_channels_after=hlval(cnt_channels_after),
cnt_channels_by_delegate_after=hlval(cnt_channels_by_delegate_after))
if stored:
# FIXME: publish WAMP event
self.log.info(
'{event}: new channel stored in database [actor_adr={actor_adr}, channel_type={channel_type}, market_oid={market_oid}, member_oid={member_oid}, channel_oid={channel_oid}]',
event=hlcontract('XBRChannel.Opened'),
market_oid=hlid(market_oid),
# FIXME
member_oid=hlid(None),
channel_oid=hlid(channel_oid),
actor_adr=hlid('0x' + binascii.b2a_hex(actor_adr).decode()),
channel_type=hlid(channel_type))
def _process_Channel_Closing(transactionHash, blockHash, args):
self.log.warn('_process_Channel_Closing not implemented')
def _process_Channel_Closed(transactionHash, blockHash, args):
self.log.warn('_process_Channel_Closed not implemented')
# map XBR contract log event to event processing function
Events = [
(xbr.xbrtoken.events.Transfer, _process_Token_Transfer),
(xbr.xbrtoken.events.Approval, _process_Token_Approval),
(xbr.xbrnetwork.events.MemberRegistered, _process_Network_MemberRegistered),
(xbr.xbrnetwork.events.MemberRetired, _process_Network_MemberRetired),
(xbr.xbrmarket.events.MarketCreated, _process_Market_MarketCreated),
(xbr.xbrmarket.events.MarketUpdated, _process_Market_MarketUpdated),
(xbr.xbrmarket.events.MarketClosed, _process_Market_MarketClosed),
(xbr.xbrmarket.events.ActorJoined, _process_Market_ActorJoined),
(xbr.xbrmarket.events.ActorLeft, _process_Market_ActorLeft),
(xbr.xbrmarket.events.ConsentSet, _process_Market_ConsentSet),
(xbr.xbrcatalog.events.CatalogCreated, _process_Catalog_CatalogCreated),
(xbr.xbrcatalog.events.ApiPublished, _process_Catalog_ApiPublished),
(xbr.xbrchannel.events.Opened, _process_Channel_Opened),
(xbr.xbrchannel.events.Closing, _process_Channel_Closing),
(xbr.xbrchannel.events.Closed, _process_Channel_Closed),
]
# determine the block number, starting from which we scan the blockchain for XBR events
current = w3.eth.getBlock('latest')
last_processed = scan_from_block - 1
with self._db.begin() as txn:
for block_number in self._xbr.blocks.select(txn, return_values=False, reverse=True, limit=1):
last_processed = unpack_uint256(block_number)
if last_processed > current.number:
raise ApplicationError(
'wamp.error.invalid_argument',
'last processed block number {} (or configured "scan_from" block number) is larger than then current block number {}'
.format(last_processed, current.number))
else:
self.log.info(
'Start scanning blockchain: current block is {current_block}, last processed is {last_processed} ..',
current_block=hlval(current.number),
last_processed=hlval(last_processed + 1))
iteration = 1
while not self._stop_monitor and not self._run_monitor.is_set():
# current last block
current = w3.eth.getBlock('latest')
# track number of blocks processed
cnt_blocks_success = 0
cnt_blocks_error = 0
cnt_xbr_events = 0
# synchronize on-change changes locally by processing blockchain events
if last_processed < current.number:
while last_processed < current.number:
last_processed += 1
try:
self.log.info('Now processing blockchain block {last_processed} ..',
last_processed=hlval(last_processed))
cnt_xbr_events += self._process_block(w3, last_processed, Events)
except:
self.log.failure()
cnt_blocks_error += 1
else:
cnt_blocks_success += 1
self.log.info(
'Monitor blockchain iteration {iteration} completed: new block processed (last_processed={last_processed}, thread_id={thread_id}, period={period}, cnt_xbr_events={cnt_xbr_events}, cnt_blocks_success={cnt_blocks_success}, cnt_blocks_error={cnt_blocks_error})',
iteration=hlval(iteration),
last_processed=hlval(last_processed),
thread_id=hlval(int(threading.get_ident())),
period=hlval(period),
cnt_xbr_events=hlval(cnt_xbr_events, color='green') if cnt_xbr_events else hlval(cnt_xbr_events),
cnt_blocks_success=hlval(cnt_blocks_success, color='green')
if cnt_xbr_events else hlval(cnt_blocks_success),
cnt_blocks_error=hlval(cnt_blocks_error, color='red')
if cnt_blocks_error else hlval(cnt_blocks_error))
else:
self.log.info(
'Monitor blockchain iteration {iteration} completed: no new blocks found (last_processed={last_processed}, thread_id={thread_id}, period={period})',
iteration=hlval(iteration),
last_processed=hlval(last_processed),
thread_id=hlval(int(threading.get_ident())),
period=hlval(period))
# sleep (using normal blocking call here, as we are running on a background thread!)
self._run_monitor.wait(period)
iteration += 1
def load_config(self, configfile=None):
"""
Load the node configuration of CFC itself. The base node configuration
is built into CFC, but can be overridden (partially) and extended
by providing a regular, local node configuration file.
When such a file exists:
- and it contains a controller section, these take precedence over
the builtin values.
- and it contains a workers section (a list of workers), these are
_added_ to the builtin workers.
"""
config_source = Node.CONFIG_SOURCE_EMPTY
# 1/4: load builtin master node configuration as default
#
config_path = pkg_resources.resource_filename('crossbar',
self.DEFAULT_CONFIG_PATH)
with open(config_path) as f:
config = json.load(f)
# no need to check the builtin config (at run-time)
# self.personality.check_config(self.personality, config)
# remember config source
config_source += Node.CONFIG_SOURCE_DEFAULT
self.log.info('Built-in master node configuration loaded')
# 2/4: allow extending/overriding the node configuration
# with a local master node configuration file (eg for TLS setup)
if configfile:
config_path = os.path.abspath(os.path.join(self._cbdir,
configfile))
with open(config_path) as f:
custom_config = json.load(f)
# as an overriding config file does not need to be a fully valid config in itself,
# do _not_ check it ..
# self.personality.check_config(self.personality, custom_config)
# .. instead, we merge the config from the local file into the already
# loaded default config (see above)
config = merge_config(config, custom_config)
# remember config source
config_source += Node.CONFIG_SOURCE_LOCALFILE
self.log.info(
'Local master node configuration merged from "{config_path}"',
config_path=hlval(config_path))
# 3/4: allow extending/overriding the node configuration
# with a remote master node configuration file from blockchain/IPFS
if config.get('controller', {}).get('blockchain', None):
blockchain_config = config['controller']['blockchain']
gateway_config = blockchain_config['gateway']
# setup Web3 connection from blockchain gateway configuration
self._w3 = make_w3(gateway_config)
# configure new Web3 connection as provider for XBR
xbr.setProvider(self._w3)
if self._w3.isConnected():
self.log.info(
'Connected to Ethereum network {network} at gateway "{gateway_url}"',
network=self._w3.net.version,
gateway_url=gateway_config['http'])
# try to find node by node public key on-chain
xbr_node_id = xbr.xbrnetwork.functions.getNodeByKey(
self.key.public_key()).call()
if xbr_node_id != b'\x00' * 16:
assert (len(xbr_node_id) == 16)
# get node domain, type, configuration and license as stored on-chain
xbr_node_domain = xbr.xbrnetwork.functions.getNodeDomain(
xbr_node_id).call()
xbr_node_type = xbr.xbrnetwork.functions.getNodeType(
xbr_node_id).call()
xbr_node_config = xbr.xbrnetwork.functions.getNodeConfig(
xbr_node_id).call()
# FIXME: for testing use hard-coded "trial license" (no. 20010)
xbr_node_license = 20010
# xbr_node_license = xbr.xbrnetwork.functions.getNodeLicense(xbr_node_id).call()
self.log.info(
'Node is configured for XBR (xbr_node_id="{xbr_node_id}", xbr_node_domain="{xbr_node_domain}", xbr_node_type="{xbr_node_type}", xbr_node_license="{xbr_node_license}", xbr_node_config="{xbr_node_config}")',
xbr_node_id=hlid(
'0x' + binascii.b2a_hex(xbr_node_id).decode()),
xbr_node_domain=hlid(
'0x' + binascii.b2a_hex(xbr_node_domain).decode()),
xbr_node_type=hlid(xbr_node_type),
xbr_node_config=hlid(xbr_node_config),
xbr_node_license=hlid(xbr_node_license))
self._license = License(xbr_node_license)
self.log.info(
'Node is registered in the XBR network with license type={license}',
license=self._license.type)
# if a (hash of a) configuration is set on-chain for the master node, fetch the
# configuration content for the hash from IPFS
if xbr_node_config:
if 'IPFS_GATEWAY_URL' in os.environ:
ipfs_gateway_url = os.environ['IPFS_GATEWAY_URL']
self.log.info(
'Using explicit IPFS gateway URL {ipfs_gateway_url} from environment variable IPFS_GATEWAY_URL',
ipfs_gateway_url=hlid(ipfs_gateway_url))
else:
ipfs_gateway_url = 'https://ipfs.infura.io:5001'
self.log.info(
'Using default IPFS Infura gateway URL {ipfs_gateway_url}',
ipfs_gateway_url=hlid(ipfs_gateway_url))
ipfs_config_url = '{}/api/v0/cat?arg={}&encoding=json'.format(
ipfs_gateway_url, xbr_node_config)
resp = requests.get(ipfs_config_url)
xbr_node_config_data = resp.json()
from pprint import pprint
pprint(xbr_node_config_data)
self.personality.check_config(self.personality,
xbr_node_config_data)
if 'controller' not in xbr_node_config_data:
xbr_node_config_data['controller'] = {}
if 'id' not in xbr_node_config_data['controller']:
xbr_node_config_data['controller']['id'] = str(
uuid.UUID(bytes=xbr_node_id))
self.log.info(
'Deriving node ID {node_id} from XBR network node ID',
node_id=hlid(
xbr_node_config_data['controller']['id']))
else:
self.log.info(
'Setting node ID {node_id} from XBR network node configuration',
node_id=hlid(
xbr_node_config_data['controller']['id']))
self._config = xbr_node_config_data
self.log.info(
'Node configuration loaded from XBR network (xbr_node_id="{xbr_node_id}")',
xbr_node_id=hlid(
'0x' + binascii.b2a_hex(xbr_node_id).decode()))
else:
self.log.debug(
'There is no node configuration stored in XBR network (xbr_node_id="{xbr_node_id}")',
xbr_node_id=hlid(
'0x' + binascii.b2a_hex(xbr_node_id).decode()))
config_source = self.CONFIG_SOURCE_XBRNETWORK
else:
self.log.warn(
'Could not find node public key on blockchain yet')
else:
self.log.warn('Could not connect to Ethereum blockchain')
# 4/4: check and set the final merged master node configuration
#
self.personality.check_config(self.personality, config)
self._config = config
self.log.debug(
'Master node config after (effective after merge):\n{config}',
config=pformat(config))
return config_source, config_path