def ok(_):
self.transports[transport_id] = router_transport
if config['endpoint']['type'] == 'tcp':
endpoint = 'TCP port {}'.format(config['endpoint']['port'])
if 'portrange' in config['endpoint']:
transport_type = 'TCP/{} transport'.format(
config['endpoint']['portrange'])
else:
transport_type = 'TCP/{} transport'.format(
config['endpoint']['port'])
elif config['endpoint']['type'] == 'unix':
endpoint = 'UDS path "{}"'.format(config['endpoint']['path'])
transport_type = 'Unix domain socket transport'
else:
endpoint = 'unknown'
transport_type = 'unknown'
self.log.info(
'Router {transport_type} started as transport "{transport_id}" and listening on {endpoint}',
transport_type=hlval(transport_type),
transport_id=hlid(transport_id),
endpoint=hlval(endpoint))
topic = '{}.on_router_transport_started'.format(self._uri_prefix)
self.publish(topic, event, options=PublishOptions(exclude=caller))
return router_transport.marshal()
async def _initialize_mrealms(self, parallel=False):
self._router_realms = {}
self._container_workers = {}
with self._db.begin() as txn:
mrealms = list(self._schema.mrealms.select(txn, return_keys=False))
if mrealms:
self.log.info(
'Initializing {cnt_mrealms} management realms (parallel={parallel}) ..',
cnt_mrealms=hlval(len(mrealms)),
parallel=hlval(parallel))
if parallel:
dl = []
for mrealm in mrealms:
dl.append(self._initialize_mrealm(mrealm))
if dl:
await DeferredList(dl)
else:
for mrealm in mrealms:
await self._initialize_mrealm(mrealm)
self.log.info(
'Ok, completed initializing {cnt_mrealms} management realms.',
cnt_mrealms=hlval(len(mrealms)))
else:
self.log.info('No management realms no initialize. Done.')
def _process_Market_MarketCreated(transactionHash, blockHash, args):
# /// Event emitted when a new market was created.
# event MarketCreated (bytes16 indexed marketId, uint32 marketSeq, address owner, string terms, string meta,
# address maker, uint256 providerSecurity, uint256 consumerSecurity, uint256 marketFee);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - XBR market created with ID {market_id})',
event=hlcontract('XBRMarket.MarketCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
market_id=hlid(uuid.UUID(bytes=args.marketId)))
market_id = uuid.UUID(bytes=args.marketId)
if args.terms:
h = multihash.decode(multihash.from_b58_string(args.terms))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRMarket.MarketCreated - terms "{terms}" is not an IPFS (sha2-256) b58-encoded multihash',
terms=hlval(args.terms))
if args.meta:
h = multihash.decode(multihash.from_b58_string(args.meta))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRMarket.MarketCreated - meta "{meta}" is not an IPFS (sha2-256) b58-encoded multihash',
meta=hlval(args.meta))
stored = False
with self._db.begin(write=True) as txn:
market = self._xbr.markets[txn, market_id]
if market:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('MarketCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
market = cfxdb.xbr.market.Market()
market.market = market_id
market.timestamp = np.datetime64(time_ns(), 'ns')
# FIXME
# market.created = args.created
market.seq = args.marketSeq
market.owner = bytes(HexBytes(args.owner))
market.terms = args.terms
market.meta = args.meta
market.maker = bytes(HexBytes(args.maker))
market.provider_security = args.providerSecurity
market.consumer_security = args.consumerSecurity
market.market_fee = args.marketFee
self._xbr.markets[txn, market_id] = market
stored = True
if stored:
self.log.info('new {contract}(market_id={market_id}) record stored database!',
contract=hlcontract('MarketCreated'),
market_id=hlid(market_id))
def _process_Network_MemberRegistered(transactionHash, blockHash, args):
# /// Event emitted when a new member joined the XBR Network.
# event MemberCreated (address indexed member, string eula, string profile, MemberLevel level);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - XBR member created at address {address})',
event=hlcontract('XBRNetwork.MemberCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
address=hlid(args.member))
member_adr = bytes(HexBytes(args.member))
if args.eula:
h = multihash.decode(multihash.from_b58_string(args.eula))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRNetwork.MemberCreated - eula "{eula}" is not an IPFS (sha2-256) b58-encoded multihash',
eula=hlval(args.eula))
if args.profile:
h = multihash.decode(multihash.from_b58_string(args.profile))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRNetwork.MemberCreated - profile "{profile}" is not an IPFS (sha2-256) b58-encoded multihash',
eula=hlval(args.profile))
stored = False
with self._db.begin(write=True) as txn:
member = self._xbr.members[txn, member_adr]
if member:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('TokenApproval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
member = cfxdb.xbr.member.Member()
member.address = member_adr
member.timestamp = np.datetime64(time_ns(), 'ns')
member.registered = args.registered
member.eula = args.eula
member.profile = args.profile
member.level = args.level
self._xbr.members[txn, member_adr] = member
stored = True
if stored:
self.log.info('new {contract}(member_adr={member_adr}) record stored database!',
contract=hlcontract('MemberCreated'),
member_adr=hlid('0x' + binascii.b2a_hex(member_adr).decode()))
def detach(self, session=None):
self.log.debug('{func}(session={session})',
func=hltype(self.detach),
session=session)
detached_session_ids = []
if session is None:
# detach all sessions from router
for session in list(self._session_id_to_session.values()):
self._detach(session)
detached_session_ids.append(session._session_id)
else:
# detach single session from router
self._detach(session)
detached_session_ids.append(session._session_id)
self.log.info(
'detached session {session} from realm "{realm}" (authid="{authid}", authrole="{authrole}", detached {detached_session_ids} sessions total) {func}',
func=hltype(self.detach),
session=hlid(session._session_id) if session else '',
authid=hlid(session._authid),
authrole=hlid(session._authrole),
detached_session_ids=hlval(len(detached_session_ids)),
realm=hlid(session._realm))
return detached_session_ids
def has_role(self, realm: str, authrole: str) -> bool:
"""
Check if a role with the given name is currently running in the given realm.
:param realm: WAMP realm (name, _not_ run-time ID).
:type realm: str
:param authrole: WAMP authentication role (URI, _not_ run-time ID).
:type authrole: str
:returns: True if realm is running.
:rtype: bool
"""
authrole = authrole or 'trusted'
result = realm in self.realm_to_id and self.realm_to_id[
realm] in self.realms
if result:
realm_id = self.realm_to_id[realm]
result = (authrole in self.realms[realm_id].role_to_id
and self.realms[realm_id].role_to_id[authrole]
in self.realms[realm_id].roles)
# note: this is to enable eg built-in "trusted" authrole
result = result or authrole in self._service_sessions[realm]
self.log.debug(
'{func}(realm="{realm}", authrole="{authrole}") -> {result}',
func=hltype(RouterController.has_role),
realm=hlid(realm),
authrole=hlid(authrole),
result=hlval(result))
return result
def hello(self, realm, details):
self.log.debug('{func}(realm="{realm}", details={details})',
func=hltype(self.hello),
realm=hlval(realm),
details=details)
extra = details.authextra or {}
for attr in ['proxy_authid', 'proxy_authrole', 'proxy_realm']:
if attr not in extra:
return types.Deny(message='missing required attribute {}'.format(attr))
realm = extra['proxy_realm']
details.authid = extra['proxy_authid']
details.authrole = extra['proxy_authrole']
details.authextra = extra.get('proxy_authextra', None)
# remember the realm the client requested to join (if any)
self._realm = realm
self._authid = details.authid
self._session_details['authmethod'] = 'anonymous'
self._session_details['authextra'] = details.authextra
self._authprovider = 'static'
# FIXME: if cookie tracking is enabled, set authid to cookie value
# self._authid = self._transport._cbtid
principal = {'realm': realm, 'authid': details.authid, 'role': details.authrole, 'extra': details.authextra}
error = self._assign_principal(principal)
if error:
return error
return self._accept()
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
regs = yield self.register(
self,
prefix='{}.'.format(self._uri_prefix),
options=RegisterOptions(details_arg='details'),
)
procs = []
errors = []
for reg in regs:
if isinstance(reg, Failure):
self.log.error("Failed to register management procedure: {f}",
f=reg,
log_failure=reg)
errors.append(str(reg))
else:
procs.append(reg.procedure)
if errors:
raise ApplicationError('crossbar.error.cannot_start',
'management API could not be initialized',
errors=errors)
else:
self.log.debug(
'Ok, registered {len_reg} management procedures on realm "{realm}" [{func}]:\n\n{procs}\n',
len_reg=hlval(len(regs)),
realm=hl(self.realm),
func=hltype(self.onJoin),
procs=hl(pformat(procs), color='white', bold=True))
returnValue(regs)
def restart_component():
# Think: if this below start_component() fails,
# we'll still schedule *exactly one* new re-start
# attempt for it, right?
self.log.info(
'{func}: now restarting previously closed component {component_id} automatically .. [restart_mode={restart_mode}, was_clean={was_clean}]',
func=hltype(_component_closed),
component_id=hlid(component_id),
restart_mode=hlval(self._restart_mode),
was_clean=hlval(was_clean))
return self.start_component(
component_id,
config,
reload_modules=reload_modules,
details=details,
)
def _process_block(self, w3, block_number, Events):
"""
:param w3:
:param block_number:
:param Events:
:return:
"""
cnt = 0
# filter by block, and XBR contract addresses
# FIXME: potentially add filters for global data or market specific data for the markets started in this worker
filter_params = {
'address': [
xbr.xbrtoken.address, xbr.xbrnetwork.address, xbr.xbrcatalog.address, xbr.xbrmarket.address,
xbr.xbrchannel.address
],
'fromBlock':
block_number,
'toBlock':
block_number,
}
result = w3.eth.getLogs(filter_params)
if result:
for evt in result:
receipt = w3.eth.getTransactionReceipt(evt['transactionHash'])
for Event, handler in Events:
# FIXME: MismatchedABI pops up .. we silence this with errors=web3.logs.DISCARD
if hasattr(web3, 'logs') and web3.logs:
all_res = Event().processReceipt(receipt, errors=web3.logs.DISCARD)
else:
all_res = Event().processReceipt(receipt)
for res in all_res:
self.log.info('{handler} processing block {block_number} / txn {txn} with args {args}',
handler=hl(handler.__name__),
block_number=hlid(block_number),
txn=hlid('0x' + binascii.b2a_hex(evt['transactionHash']).decode()),
args=hlval(res.args))
handler(res.transactionHash, res.blockHash, res.args)
cnt += 1
with self._db.begin(write=True) as txn:
block = cfxdb.xbr.block.Block()
block.timestamp = np.datetime64(time_ns(), 'ns')
block.block_number = block_number
# FIXME
# block.block_hash = bytes()
block.cnt_events = cnt
self._xbr.blocks[txn, pack_uint256(block_number)] = block
if cnt:
self.log.info('Processed blockchain block {block_number}: processed {cnt} XBR events.',
block_number=hlid(block_number),
cnt=hlid(cnt))
else:
self.log.info('Processed blockchain block {block_number}: no XBR events found!',
block_number=hlid(block_number))
return cnt
def stop_web_transport_service(self, transport_id, path, details=None):
"""
Stop a service on a Web transport.
:param transport_id: The ID of the transport to stop the Web transport service on.
:type transport_id: str
:param path: The path (absolute URL, eg "/myservice1") of the service to stop.
:type path: str
:param details: Call details.
:type details: :class:`autobahn.wamp.types.CallDetails`
"""
self.log.info('{func}(transport_id={transport_id}, path="{path}")',
func=hltype(self.stop_web_transport_service),
transport_id=hlid(transport_id),
path=hlval(path))
transport = self.transports.get(transport_id, None)
if not transport or \
not isinstance(transport, self.personality.RouterWebTransport) or \
transport.state != self.personality.RouterTransport.STATE_STARTED:
emsg = "Cannot stop service on Web transport: no transport with ID '{}' or transport is not a Web transport".format(
transport_id)
self.log.error(emsg)
raise ApplicationError('crossbar.error.not_running', emsg)
if path not in transport.root:
emsg = "Cannot stop service on Web transport {}: no service running on path '{}'".format(
transport_id, path)
self.log.error(emsg)
raise ApplicationError('crossbar.error.not_running', emsg)
caller = details.caller if details else None
self.publish(self._uri_prefix + '.on_web_transport_service_stopping',
transport_id,
path,
options=PublishOptions(exclude=caller))
# now actually remove the web service. note: currently this is NOT async, but direct/sync.
# FIXME: check that the underlying Twisted Web resource doesn't need any stopping too!
del transport.root[path]
on_web_transport_service_stopped = {
'transport_id': transport_id,
'path': path,
}
caller = details.caller if details else None
self.publish(self._uri_prefix + '.on_web_transport_service_stopped',
transport_id,
path,
on_web_transport_service_stopped,
options=PublishOptions(exclude=caller))
return on_web_transport_service_stopped
def _process_Market_ActorJoined(transactionHash, blockHash, args):
# Event emitted when a new actor joined a market.
# event ActorJoined (bytes16 indexed marketId, address actor, uint8 actorType, uint joined, uint256 security, string meta);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - XBR market actor {actor} joined market {market_id})',
event=hlcontract('XBRMarket.ActorJoined'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
actor=hlid(args.actor),
market_id=hlid(uuid.UUID(bytes=args.marketId)))
market_id = uuid.UUID(bytes=args.marketId)
actor_adr = bytes(HexBytes(args.actor))
actor_type = int(args.actorType)
if args.meta:
h = multihash.decode(multihash.from_b58_string(args.meta))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRMarket.MarketCreated - meta "{meta}" is not an IPFS (sha2-256) b58-encoded multihash',
terms=hlval(args.meta))
stored = False
with self._db.begin(write=True) as txn:
actor = self._xbr.actors[txn, (market_id, actor_adr, actor_type)]
if actor:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('MarketCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
actor = cfxdb.xbr.actor.Actor()
actor.timestamp = np.datetime64(time_ns(), 'ns')
actor.market = market_id
actor.actor = actor_adr
actor.actor_type = actor_type
actor.joined = args.joined
actor.security = args.security
actor.meta = args.meta
self._xbr.actors[txn, (market_id, actor_adr, actor_type)] = actor
stored = True
if stored:
self.log.info(
'new {contract}(market_id={market_id}, actor_adr={actor_adr}, actor_type={actor_type}) record stored database!',
contract=hlcontract('ActorJoined'),
market_id=hlid(market_id),
actor_adr=hlid('0x' + binascii.b2a_hex(actor_adr).decode()),
actor_type=hlid(actor_type))
def _process_Token_Approval(transactionHash, blockHash, args):
# event Approval(address indexed from, address indexed to, uint256 value);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - {value} XBR token approved (on-chain) from owner {owner} to spender {spender})',
event=hlcontract('XBRToken.Approval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
value=hlval(int(args.value / 10**18)),
owner=hlval(args.owner),
spender=hlval(args.spender))
stored = False
with self._db.begin(write=True) as txn:
transactionHash = bytes(transactionHash)
token_approval = self._xbr.token_approvals[txn, transactionHash]
if token_approval:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('TokenApproval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
token_approval = cfxdb.xbr.token.TokenApproval()
token_approval.tx_hash = transactionHash
token_approval.block_hash = bytes(blockHash)
token_approval.owner_address = bytes(HexBytes(args.owner))
token_approval.spender_address = bytes(HexBytes(args.spender))
token_approval.value = args.value
self._xbr.token_approvals[txn, transactionHash] = token_approval
stored = True
if stored:
self.log.info('new {contract}(tx_hash={tx_hash}) record stored database!',
contract=hlcontract('TokenApproval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
def _process_Token_Transfer(transactionHash, blockHash, args):
# event Transfer(address indexed from, address indexed to, uint256 value);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - {value} XBR token transferred (on-chain) from {_from} to {_to})',
event=hlcontract('XBRToken.Transfer'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
value=hlval(int(args.value / 10**18)),
_from=hlval(args['from']),
_to=hlval(args.to))
stored = False
with self._db.begin(write=True) as txn:
transactionHash = bytes(transactionHash)
token_transfer = self._xbr.token_transfers[txn, transactionHash]
if token_transfer:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('TokenTransfer'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
token_transfer = cfxdb.xbr.token.TokenTransfer()
token_transfer.tx_hash = transactionHash
token_transfer.block_hash = bytes(blockHash)
token_transfer.from_address = bytes(HexBytes(args['from']))
token_transfer.to_address = bytes(HexBytes(args.to))
token_transfer.value = args.value
self._xbr.token_transfers[txn, transactionHash] = token_transfer
stored = True
if stored:
self.log.info('new {contract}(tx_hash={tx_hash}) record stored database!',
contract=hlcontract('TokenTransfer'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
def has_realm(self, realm: str) -> bool:
"""
Check if a realm with the given name is currently running.
:param realm: Realm name (_not_ ID).
:type realm: str
:returns: True if realm is running.
:rtype: bool
"""
result = realm in self.realm_to_id and self.realm_to_id[
realm] in self.realms
self.log.debug('{func}(realm="{realm}") -> {result}',
func=hltype(RouterController.has_realm),
realm=hlid(realm),
result=hlval(result))
return result
def get_web_transport_service(self, transport_id, path, details=None):
self.log.debug('{func}(transport_id={transport_id}, path="{path}")',
func=hltype(self.get_web_transport_service),
transport_id=hlid(transport_id),
path=hlval(path))
transport = self.transports.get(transport_id, None)
if not transport or \
not isinstance(transport, self.personality.RouterWebTransport) or \
transport.state != self.personality.RouterTransport.STATE_STARTED:
emsg = "No transport with ID '{}' or transport is not a Web transport".format(transport_id)
self.log.debug(emsg)
raise ApplicationError('crossbar.error.not_running', emsg)
if path not in transport.root:
emsg = "Web transport {}: no service running on path '{}'".format(transport_id, path)
self.log.debug(emsg)
raise ApplicationError('crossbar.error.not_running', emsg)
return transport.marshal()
def onJoin(self, details):
"""
Called when process has joined the node's management realm.
"""
regs = yield self.register(
self,
prefix='{}.'.format(self._uri_prefix),
options=RegisterOptions(details_arg='details'),
)
self.log.info(
'Ok, registered {len_reg} management procedures on realm "{realm}".',
len_reg=hlval(len(regs)),
realm=hl(self.realm))
for reg in regs:
if isinstance(reg, Failure):
self.log.error("Failed to register: {f}",
f=reg,
log_failure=reg)
else:
self.log.debug(' {proc}', proc=hlid(reg.procedure))
returnValue(regs)
def start_router_realm_link(self,
realm_id,
link_id,
link_config,
details=None):
"""
Start a new router link to a remote router on a (local) realm.
The link configuration (``link_config``) must include the transport definition
to the remote router. Here is an example:
.. code-block:: json
{
"realm": "realm1",
"transport": {
"type": "websocket",
"endpoint": {
"type": "tcp",
"host": "localhost",
"port": 8002
},
"url": "ws://localhost:8002/ws"
}
}
:param realm_id: The ID of the (local) realm on which to start the link.
:type realm_id: str
:param link_id: The ID of the router link to start.
:type link_id: str
:param link_config: The router link configuration.
:type link_config: dict
:returns: The new link detail information.
:rtype: dict
"""
assert type(realm_id) == str
assert type(link_id) == str
assert type(link_config) == dict
assert isinstance(details, CallDetails)
self.log.info(
'{method} Router link {link_id} starting on realm {realm_id} ..',
link_id=hlid(link_id),
realm_id=hlid(realm_id),
method=hltype(RouterController.start_router_realm_link))
try:
if realm_id not in self.realms:
self.log.warn('{func} realm "{realm}" not found in {realms}',
func=hltype(self.start_router_realm_link),
realm=hlval(realm_id),
realms=sorted(self.realms.keys()))
raise ApplicationError('crossbar.error.no_such_object',
'no realm with ID {}'.format(realm_id))
rlink_manager = self.realms[realm_id].rlink_manager
if link_id in rlink_manager:
raise ApplicationError(
'crossbar.error.already_running',
'router link {} already running'.format(link_id))
link_config = RLinkConfig.parse(self.personality,
link_config,
id=link_id)
caller = SessionIdent.from_calldetails(details)
rlink = yield rlink_manager.start_link(link_id, link_config,
caller)
started = rlink.marshal()
except:
self.log.failure()
raise
else:
self.publish(
'{}.on_router_realm_link_started'.format(self._uri_prefix),
started)
self.log.info('Router link {link_id} started on realm {realm_id}',
link_id=hlid(link_id),
realm_id=hlid(realm_id))
returnValue(started)
def _monitor_blockchain(self, gateway_config, scan_from_block, period=300):
"""
:param gateway_config:
:param scan_from_block:
:param period:
:return:
"""
w3 = make_w3(gateway_config)
initial_delay = 2
self.log.info(
'Start monitoring of blockchain ({blockchain_type}) on thread {thread_id} in {initial_delay} seconds, iterating every {period} seconds ..',
blockchain_type=str(self._bc_gw_config['type']),
initial_delay=hlval(initial_delay),
period=hlval(period),
thread_id=hlval(int(threading.get_ident())))
# using normal blocking call here, as we are running on a background thread!
time.sleep(initial_delay)
def _process_Token_Transfer(transactionHash, blockHash, args):
# event Transfer(address indexed from, address indexed to, uint256 value);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - {value} XBR token transferred (on-chain) from {_from} to {_to})',
event=hlcontract('XBRToken.Transfer'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
value=hlval(int(args.value / 10**18)),
_from=hlval(args['from']),
_to=hlval(args.to))
stored = False
with self._db.begin(write=True) as txn:
transactionHash = bytes(transactionHash)
token_transfer = self._xbr.token_transfers[txn, transactionHash]
if token_transfer:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('TokenTransfer'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
token_transfer = cfxdb.xbr.token.TokenTransfer()
token_transfer.tx_hash = transactionHash
token_transfer.block_hash = bytes(blockHash)
token_transfer.from_address = bytes(HexBytes(args['from']))
token_transfer.to_address = bytes(HexBytes(args.to))
token_transfer.value = args.value
self._xbr.token_transfers[txn, transactionHash] = token_transfer
stored = True
if stored:
self.log.info('new {contract}(tx_hash={tx_hash}) record stored database!',
contract=hlcontract('TokenTransfer'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
def _process_Token_Approval(transactionHash, blockHash, args):
# event Approval(address indexed from, address indexed to, uint256 value);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - {value} XBR token approved (on-chain) from owner {owner} to spender {spender})',
event=hlcontract('XBRToken.Approval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
value=hlval(int(args.value / 10**18)),
owner=hlval(args.owner),
spender=hlval(args.spender))
stored = False
with self._db.begin(write=True) as txn:
transactionHash = bytes(transactionHash)
token_approval = self._xbr.token_approvals[txn, transactionHash]
if token_approval:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('TokenApproval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
token_approval = cfxdb.xbr.token.TokenApproval()
token_approval.tx_hash = transactionHash
token_approval.block_hash = bytes(blockHash)
token_approval.owner_address = bytes(HexBytes(args.owner))
token_approval.spender_address = bytes(HexBytes(args.spender))
token_approval.value = args.value
self._xbr.token_approvals[txn, transactionHash] = token_approval
stored = True
if stored:
self.log.info('new {contract}(tx_hash={tx_hash}) record stored database!',
contract=hlcontract('TokenApproval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
def _process_Network_MemberRegistered(transactionHash, blockHash, args):
# /// Event emitted when a new member joined the XBR Network.
# event MemberCreated (address indexed member, string eula, string profile, MemberLevel level);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - XBR member created at address {address})',
event=hlcontract('XBRNetwork.MemberCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
address=hlid(args.member))
member_adr = bytes(HexBytes(args.member))
if args.eula:
h = multihash.decode(multihash.from_b58_string(args.eula))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRNetwork.MemberCreated - eula "{eula}" is not an IPFS (sha2-256) b58-encoded multihash',
eula=hlval(args.eula))
if args.profile:
h = multihash.decode(multihash.from_b58_string(args.profile))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRNetwork.MemberCreated - profile "{profile}" is not an IPFS (sha2-256) b58-encoded multihash',
eula=hlval(args.profile))
stored = False
with self._db.begin(write=True) as txn:
member = self._xbr.members[txn, member_adr]
if member:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('TokenApproval'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
member = cfxdb.xbr.member.Member()
member.address = member_adr
member.timestamp = np.datetime64(time_ns(), 'ns')
member.registered = args.registered
member.eula = args.eula
member.profile = args.profile
member.level = args.level
self._xbr.members[txn, member_adr] = member
stored = True
if stored:
self.log.info('new {contract}(member_adr={member_adr}) record stored database!',
contract=hlcontract('MemberCreated'),
member_adr=hlid('0x' + binascii.b2a_hex(member_adr).decode()))
def _process_Network_MemberRetired(transactionHash, blockHash, args):
# /// Event emitted when a member leaves the XBR Network.
# event MemberRetired (address member);
self.log.warn('_process_Network_MemberRetired not implemented')
def _process_Market_MarketCreated(transactionHash, blockHash, args):
# /// Event emitted when a new market was created.
# event MarketCreated (bytes16 indexed marketId, uint32 marketSeq, address owner, string terms, string meta,
# address maker, uint256 providerSecurity, uint256 consumerSecurity, uint256 marketFee);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - XBR market created with ID {market_id})',
event=hlcontract('XBRMarket.MarketCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
market_id=hlid(uuid.UUID(bytes=args.marketId)))
market_id = uuid.UUID(bytes=args.marketId)
if args.terms:
h = multihash.decode(multihash.from_b58_string(args.terms))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRMarket.MarketCreated - terms "{terms}" is not an IPFS (sha2-256) b58-encoded multihash',
terms=hlval(args.terms))
if args.meta:
h = multihash.decode(multihash.from_b58_string(args.meta))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRMarket.MarketCreated - meta "{meta}" is not an IPFS (sha2-256) b58-encoded multihash',
meta=hlval(args.meta))
stored = False
with self._db.begin(write=True) as txn:
market = self._xbr.markets[txn, market_id]
if market:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('MarketCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
market = cfxdb.xbr.market.Market()
market.market = market_id
market.timestamp = np.datetime64(time_ns(), 'ns')
# FIXME
# market.created = args.created
market.seq = args.marketSeq
market.owner = bytes(HexBytes(args.owner))
market.terms = args.terms
market.meta = args.meta
market.maker = bytes(HexBytes(args.maker))
market.provider_security = args.providerSecurity
market.consumer_security = args.consumerSecurity
market.market_fee = args.marketFee
self._xbr.markets[txn, market_id] = market
stored = True
if stored:
self.log.info('new {contract}(market_id={market_id}) record stored database!',
contract=hlcontract('MarketCreated'),
market_id=hlid(market_id))
def _process_Market_MarketUpdated(transactionHash, blockHash, args):
# /// Event emitted when a market was updated.
# event MarketUpdated (bytes16 indexed marketId, uint32 marketSeq, address owner, string terms, string meta,
# address maker, uint256 providerSecurity, uint256 consumerSecurity, uint256 marketFee);
self.log.warn('_process_Market_MarketUpdated not implemented')
def _process_Market_MarketClosed(transactionHash, blockHash, args):
# /// Event emitted when a market was closed.
# event MarketClosed (bytes16 indexed marketId);
self.log.warn('_process_Market_MarketClosed not implemented')
def _process_Market_ActorJoined(transactionHash, blockHash, args):
# Event emitted when a new actor joined a market.
# event ActorJoined (bytes16 indexed marketId, address actor, uint8 actorType, uint joined, uint256 security, string meta);
self.log.info(
'{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) - XBR market actor {actor} joined market {market_id})',
event=hlcontract('XBRMarket.ActorJoined'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()),
actor=hlid(args.actor),
market_id=hlid(uuid.UUID(bytes=args.marketId)))
market_id = uuid.UUID(bytes=args.marketId)
actor_adr = bytes(HexBytes(args.actor))
actor_type = int(args.actorType)
if args.meta:
h = multihash.decode(multihash.from_b58_string(args.meta))
if h.name != 'sha2-256':
self.log.warn(
'WARNING: XBRMarket.MarketCreated - meta "{meta}" is not an IPFS (sha2-256) b58-encoded multihash',
terms=hlval(args.meta))
stored = False
with self._db.begin(write=True) as txn:
actor = self._xbr.actors[txn, (market_id, actor_adr, actor_type)]
if actor:
self.log.warn('{contract}(tx_hash={tx_hash}) record already stored in database.',
contract=hlcontract('MarketCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()))
else:
actor = cfxdb.xbr.actor.Actor()
actor.timestamp = np.datetime64(time_ns(), 'ns')
actor.market = market_id
actor.actor = actor_adr
actor.actor_type = actor_type
actor.joined = args.joined
actor.security = args.security
actor.meta = args.meta
self._xbr.actors[txn, (market_id, actor_adr, actor_type)] = actor
stored = True
if stored:
self.log.info(
'new {contract}(market_id={market_id}, actor_adr={actor_adr}, actor_type={actor_type}) record stored database!',
contract=hlcontract('ActorJoined'),
market_id=hlid(market_id),
actor_adr=hlid('0x' + binascii.b2a_hex(actor_adr).decode()),
actor_type=hlid(actor_type))
def _process_Market_ActorLeft(transactionHash, blockHash, args):
self.log.warn('_process_Market_ActorLeft not implemented')
def _process_Market_ConsentSet(transactionHash, blockHash, args):
# Event emitted when a consent is set
# emit ConsentSet(member, updated, marketId, delegate, delegateType,
# apiCatalog, consent, servicePrefix);
self.log.info('{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) ..',
event=hlcontract('XBRMarket.ConsentSet'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()))
catalog_oid = uuid.UUID(bytes=args.apiCatalog)
member = uuid.UUID(bytes=args.member)
delegate = args.delegate
delegate_type = args.delegateType
market_oid = uuid.UUID(bytes=args.marketId)
with self._db.begin(write=True) as txn:
consent = self._xbr.consents[txn, (catalog_oid, member, delegate, delegate_type, market_oid)]
consent.synced = True
def _process_Catalog_CatalogCreated(transactionHash, blockHash, args):
# Event emitted when a new API catalog is created
# emit CatalogCreated(catalogId, created, catalogSeq, owner, terms, meta);
self.log.info('{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) ..',
event=hlcontract('XBRCatalog.CatalogCreated'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()))
catalog_oid = uuid.UUID(bytes=args.catalogId)
owner = bytes(HexBytes(args.owner))
created = np.datetime64(time_ns(), 'ns')
with self._db.begin(write=True) as txn:
catalog = cfxdb.xbr.catalog.Catalog()
catalog.oid = catalog_oid
catalog.timestamp = created
catalog.seq = args.catalogSeq
catalog.owner = owner
catalog.terms = args.terms
catalog.meta = args.meta
self._xbr.catalogs[txn, catalog_oid] = catalog
deferToThread(self._download_ipfs_file, args.meta)
def _process_Catalog_ApiPublished(transactionHash, blockHash, args):
self.log.warn('_process_Catalog_ApiPublished not implemented')
def _process_Channel_Opened(transactionHash, blockHash, args):
# Event emitted when a new XBR data market has opened.
# event Opened(XBRTypes.ChannelType ctype, bytes16 indexed marketId, bytes16 indexed channelId,
# address actor, address delegate, address marketmaker, address recipient,
# uint256 amount, bytes signature);
self.log.info('{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) ..',
event=hlcontract('XBRChannel.Opened'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()))
channel_oid = uuid.UUID(bytes=args.channelId)
marketmaker_adr = bytes(HexBytes(args.marketmaker))
marketmaker_adr_str = web3.Web3.toChecksumAddress(marketmaker_adr)
# we only persist data for xbr markets operated by one of the market makers we run in this worker
if marketmaker_adr_str not in self._maker_adr2id or self._maker_adr2id[
marketmaker_adr_str] not in self._makers:
self.log.info(
'{event}: skipping channel (channel {channel_oid} in market with market maker address {marketmaker_adr} is in for any market in this markets worker)',
event=hlcontract('XBRChannel.Opened'),
channel_oid=hlid(channel_oid),
marketmaker_adr=hlid(marketmaker_adr_str))
return
# prepare new channel data
channel_type = int(args.ctype)
market_oid = uuid.UUID(bytes=args.marketId)
actor_adr = bytes(HexBytes(args.actor))
delegate_adr = bytes(HexBytes(args.delegate))
recipient_adr = bytes(HexBytes(args.recipient))
amount = int(args.amount)
# FIXME
# signature = bytes(HexBytes(args.signature))
# FIXME
# member_oid = uuid.UUID()
# get the market maker by address
maker = self._makers[self._maker_adr2id[marketmaker_adr_str]]
# the market maker specific embedded database and schema
db = maker.db
xbrmm = maker.schema
# depending on channel type, different database schema classes and tables are used
if channel_type == cfxdb.xbrmm.ChannelType.PAYMENT:
channel = cfxdb.xbrmm.PaymentChannel()
channels = xbrmm.payment_channels
channels_by_delegate = xbrmm.idx_payment_channel_by_delegate
balance = cfxdb.xbrmm.PaymentChannelBalance()
balances = xbrmm.payment_balances
elif channel_type == cfxdb.xbrmm.ChannelType.PAYING:
channel = cfxdb.xbrmm.PayingChannel()
channels = xbrmm.paying_channels
channels_by_delegate = xbrmm.idx_paying_channel_by_delegate
balance = cfxdb.xbrmm.PayingChannelBalance()
balances = xbrmm.paying_balances
else:
assert False, 'should not arrive here'
# fill in information for newly replicated channel
channel.market_oid = market_oid
# FIXME
# channel.member_oid = member_oid
channel.channel_oid = channel_oid
channel.timestamp = np.datetime64(time_ns(), 'ns')
# channel.open_at = None
# FIXME: should read that from even args after deployment of
# https://github.com/crossbario/xbr-protocol/pull/138
channel.seq = 1
channel.channel_type = channel_type
channel.marketmaker = marketmaker_adr
channel.actor = actor_adr
channel.delegate = delegate_adr
channel.recipient = recipient_adr
channel.amount = amount
# FIXME
channel.timeout = 0
channel.state = cfxdb.xbrmm.ChannelState.OPEN
# FIXME
# channel.open_sig = signature
# create an off-chain balance record for the channel with remaining == initial amount
balance.remaining = channel.amount
# FIXME: should read that from even args after deployment of
# https://github.com/crossbario/xbr-protocol/pull/138
balance.seq = 1
# now store the new channel and balance in the database
stored = False
cnt_channels_before = 0
cnt_channels_by_delegate_before = 0
cnt_channels_after = 0
cnt_channels_by_delegate_after = 0
with db.begin(write=True) as txn:
if channels[txn, channel_oid]:
self.log.warn('{event}: channel already stored in database [channel_oid={channel_oid}]',
event=hlcontract('XBRChannel.Opened'),
channel_oid=hlid(channel_oid))
else:
cnt_channels_before = channels.count(txn)
cnt_channels_by_delegate_before = channels_by_delegate.count(txn)
# store the channel along with the off-chain balance
channels[txn, channel_oid] = channel
balances[txn, channel_oid] = balance
stored = True
cnt_channels_after = channels.count(txn)
cnt_channels_by_delegate_after = channels_by_delegate.count(txn)
self.log.info(
'{event} DB result: stored={stored}, cnt_channels_before={cnt_channels_before}, cnt_channels_by_delegate_before={cnt_channels_by_delegate_before}, cnt_channels_after={cnt_channels_after}, cnt_channels_by_delegate_after={cnt_channels_by_delegate_after}',
event=hlcontract('XBRChannel.Opened'),
stored=hlval(stored),
cnt_channels_before=hlval(cnt_channels_before),
cnt_channels_by_delegate_before=hlval(cnt_channels_by_delegate_before),
cnt_channels_after=hlval(cnt_channels_after),
cnt_channels_by_delegate_after=hlval(cnt_channels_by_delegate_after))
if stored:
# FIXME: publish WAMP event
self.log.info(
'{event}: new channel stored in database [actor_adr={actor_adr}, channel_type={channel_type}, market_oid={market_oid}, member_oid={member_oid}, channel_oid={channel_oid}]',
event=hlcontract('XBRChannel.Opened'),
market_oid=hlid(market_oid),
# FIXME
member_oid=hlid(None),
channel_oid=hlid(channel_oid),
actor_adr=hlid('0x' + binascii.b2a_hex(actor_adr).decode()),
channel_type=hlid(channel_type))
def _process_Channel_Closing(transactionHash, blockHash, args):
self.log.warn('_process_Channel_Closing not implemented')
def _process_Channel_Closed(transactionHash, blockHash, args):
self.log.warn('_process_Channel_Closed not implemented')
# map XBR contract log event to event processing function
Events = [
(xbr.xbrtoken.events.Transfer, _process_Token_Transfer),
(xbr.xbrtoken.events.Approval, _process_Token_Approval),
(xbr.xbrnetwork.events.MemberRegistered, _process_Network_MemberRegistered),
(xbr.xbrnetwork.events.MemberRetired, _process_Network_MemberRetired),
(xbr.xbrmarket.events.MarketCreated, _process_Market_MarketCreated),
(xbr.xbrmarket.events.MarketUpdated, _process_Market_MarketUpdated),
(xbr.xbrmarket.events.MarketClosed, _process_Market_MarketClosed),
(xbr.xbrmarket.events.ActorJoined, _process_Market_ActorJoined),
(xbr.xbrmarket.events.ActorLeft, _process_Market_ActorLeft),
(xbr.xbrmarket.events.ConsentSet, _process_Market_ConsentSet),
(xbr.xbrcatalog.events.CatalogCreated, _process_Catalog_CatalogCreated),
(xbr.xbrcatalog.events.ApiPublished, _process_Catalog_ApiPublished),
(xbr.xbrchannel.events.Opened, _process_Channel_Opened),
(xbr.xbrchannel.events.Closing, _process_Channel_Closing),
(xbr.xbrchannel.events.Closed, _process_Channel_Closed),
]
# determine the block number, starting from which we scan the blockchain for XBR events
current = w3.eth.getBlock('latest')
last_processed = scan_from_block - 1
with self._db.begin() as txn:
for block_number in self._xbr.blocks.select(txn, return_values=False, reverse=True, limit=1):
last_processed = unpack_uint256(block_number)
if last_processed > current.number:
raise ApplicationError(
'wamp.error.invalid_argument',
'last processed block number {} (or configured "scan_from" block number) is larger than then current block number {}'
.format(last_processed, current.number))
else:
self.log.info(
'Start scanning blockchain: current block is {current_block}, last processed is {last_processed} ..',
current_block=hlval(current.number),
last_processed=hlval(last_processed + 1))
iteration = 1
while not self._stop_monitor and not self._run_monitor.is_set():
# current last block
current = w3.eth.getBlock('latest')
# track number of blocks processed
cnt_blocks_success = 0
cnt_blocks_error = 0
cnt_xbr_events = 0
# synchronize on-change changes locally by processing blockchain events
if last_processed < current.number:
while last_processed < current.number:
last_processed += 1
try:
self.log.info('Now processing blockchain block {last_processed} ..',
last_processed=hlval(last_processed))
cnt_xbr_events += self._process_block(w3, last_processed, Events)
except:
self.log.failure()
cnt_blocks_error += 1
else:
cnt_blocks_success += 1
self.log.info(
'Monitor blockchain iteration {iteration} completed: new block processed (last_processed={last_processed}, thread_id={thread_id}, period={period}, cnt_xbr_events={cnt_xbr_events}, cnt_blocks_success={cnt_blocks_success}, cnt_blocks_error={cnt_blocks_error})',
iteration=hlval(iteration),
last_processed=hlval(last_processed),
thread_id=hlval(int(threading.get_ident())),
period=hlval(period),
cnt_xbr_events=hlval(cnt_xbr_events, color='green') if cnt_xbr_events else hlval(cnt_xbr_events),
cnt_blocks_success=hlval(cnt_blocks_success, color='green')
if cnt_xbr_events else hlval(cnt_blocks_success),
cnt_blocks_error=hlval(cnt_blocks_error, color='red')
if cnt_blocks_error else hlval(cnt_blocks_error))
else:
self.log.info(
'Monitor blockchain iteration {iteration} completed: no new blocks found (last_processed={last_processed}, thread_id={thread_id}, period={period})',
iteration=hlval(iteration),
last_processed=hlval(last_processed),
thread_id=hlval(int(threading.get_ident())),
period=hlval(period))
# sleep (using normal blocking call here, as we are running on a background thread!)
self._run_monitor.wait(period)
iteration += 1
def start_web_transport_service(self,
transport_id,
path,
config,
details=None):
"""
Start a service on a Web transport.
:param transport_id: The ID of the transport to start the Web transport service on.
:type transport_id: str
:param path: The path (absolute URL, eg "/myservice1") on which to start the service.
:type path: str
:param config: The Web service configuration.
:type config: dict
:param details: Call details.
:type details: :class:`autobahn.wamp.types.CallDetails`
"""
if not isinstance(config, dict) or 'type' not in config:
raise ApplicationError(
'crossbar.invalid_argument',
'config parameter must be dict with type attribute')
self.log.info(
'Starting "{service_type}" Web service on path "{path}" of transport "{transport_id}" {func}',
service_type=hlval(config.get('type', 'unknown')),
path=hlval(path),
transport_id=hlid(transport_id),
func=hltype(self.start_web_transport_service))
transport = self.transports.get(transport_id, None)
if not transport:
emsg = 'Cannot start service on transport: no transport with ID "{}"'.format(
transport_id)
self.log.error(emsg)
raise ApplicationError('crossbar.error.not_running', emsg)
if not isinstance(transport, self.personality.RouterWebTransport):
emsg = 'Cannot start service on transport: transport is not a Web transport (transport_type={})'.format(
hltype(transport.__class__))
self.log.error(emsg)
raise ApplicationError('crossbar.error.not_running', emsg)
if transport.state != self.personality.RouterTransport.STATE_STARTED:
emsg = 'Cannot start service on Web transport service: transport {} is not running (transport_state={})'.format(
transport_id,
self.personality.RouterWebTransport.STATES.get(
transport.state, None))
self.log.error(emsg)
raise ApplicationError('crossbar.error.not_running', emsg)
if path in transport.root:
emsg = 'Cannot start service on Web transport "{}": a service is already running on path "{}"'.format(
transport_id, path)
self.log.error(emsg)
raise ApplicationError('crossbar.error.already_running', emsg)
caller = details.caller if details else None
self.publish(self._uri_prefix + '.on_web_transport_service_starting',
transport_id,
path,
options=PublishOptions(exclude=caller))
# now actually add the web service ..
# note: currently this is NOT async, but direct/sync.
webservice_factory = self.personality.WEB_SERVICE_FACTORIES[
config['type']]
webservice = yield maybeDeferred(webservice_factory.create, transport,
path, config)
transport.root[path] = webservice
on_web_transport_service_started = {
'transport_id': transport_id,
'path': path,
'config': config
}
caller = details.caller if details else None
self.publish(self._uri_prefix + '.on_web_transport_service_started',
transport_id,
path,
on_web_transport_service_started,
options=PublishOptions(exclude=caller))
returnValue(on_web_transport_service_started)
def _component_closed(session, was_clean):
"""
This is moderate hack around the fact that we don't have any way
to "listen" for a close event on websocket or rawsocket
objects. Also, the rawsocket implementation doesn't have
"a" function we can wrap anyway (they are asyncio vs
Twisted specific), so for both WebSocket and rawsocket
cases, we actually listen on the WAMP session for
transport close notifications.
Ideally we'd listen for "close" on the transport but this
works fine for cleaning up the components.
"""
if component_id not in self.components:
self.log.warn(
"Component '{id}' closed, but not in set.",
id=component_id,
)
return
if was_clean:
self.log.info(
"Closed connection to '{id}'",
id=component_id,
)
else:
self.log.error(
"Lost connection to component '{id}' uncleanly",
id=component_id,
)
component = self.components[component_id]
del self.components[component_id]
self._publish_component_stop(component)
component._stopped.callback(component.marshal())
del component
# figure out if we need to shut down the container itself or not
if not was_clean and self._exit_mode == self.SHUTDOWN_ON_ANY_COMPONENT_FAILED:
self.log.info(
"A component has failed: stopping container in exit mode <{exit_mode}> ...",
exit_mode=self._exit_mode,
)
self.shutdown()
return
if self._exit_mode == self.SHUTDOWN_ON_ANY_COMPONENT_STOPPED:
self.log.info(
"A component has stopped: stopping container in exit mode <{exit_mode}> ...",
exit_mode=self._exit_mode,
)
self.shutdown()
return
if not self.components:
if self._exit_mode == self.SHUTDOWN_ON_LAST_COMPONENT_STOPPED:
self.log.info(
"Container is hosting no more components: stopping container in exit mode <{exit_mode}> ...",
exit_mode=self._exit_mode,
)
self.shutdown()
return
else:
self.log.info(
"Container is hosting no more components: continue running in exit mode <{exit_mode}>",
exit_mode=self._exit_mode,
)
else:
self.log.info(
"Container is still hosting {component_count} components: continue running in exit mode <{exit_mode}>",
exit_mode=self._exit_mode,
component_count=len(self.components),
)
# determine if we should re-start the component. Note that
# we can only arrive here if we *didn't* decide to
# shutdown above .. so if we have a shutdown mode of
# SHUTDOWN_ON_ANY_COMPONENT_STOPPED will mean we never try
# to re-start anything.
if self._restart_mode == self.RESTART_ALWAYS or (
self._restart_mode == self.RESTART_FAILED
and not was_clean):
def restart_component():
# Think: if this below start_component() fails,
# we'll still schedule *exactly one* new re-start
# attempt for it, right?
self.log.info(
'{func}: now restarting previously closed component {component_id} automatically .. [restart_mode={restart_mode}, was_clean={was_clean}]',
func=hltype(_component_closed),
component_id=hlid(component_id),
restart_mode=hlval(self._restart_mode),
was_clean=hlval(was_clean))
return self.start_component(
component_id,
config,
reload_modules=reload_modules,
details=details,
)
# note we must yield to the reactor with
# callLater(0, ..) to avoid infinite recursion if
# we're stuck in a restart loop
from twisted.internet import reactor
reactor.callLater(0, restart_component)
else:
self.log.warn(
'{func}: component {component_id} will not be restarted automatically! [restart_mode={restart_mode}, was_clean={was_clean}]',
func=hltype(_component_closed),
component_id=hlid(component_id),
restart_mode=hlval(self._restart_mode),
was_clean=hlval(was_clean))
def _process_Channel_Opened(transactionHash, blockHash, args):
# Event emitted when a new XBR data market has opened.
# event Opened(XBRTypes.ChannelType ctype, bytes16 indexed marketId, bytes16 indexed channelId,
# address actor, address delegate, address marketmaker, address recipient,
# uint256 amount, bytes signature);
self.log.info('{event}: processing event (tx_hash={tx_hash}, block_hash={block_hash}) ..',
event=hlcontract('XBRChannel.Opened'),
tx_hash=hlid('0x' + binascii.b2a_hex(transactionHash).decode()),
block_hash=hlid('0x' + binascii.b2a_hex(blockHash).decode()))
channel_oid = uuid.UUID(bytes=args.channelId)
marketmaker_adr = bytes(HexBytes(args.marketmaker))
marketmaker_adr_str = web3.Web3.toChecksumAddress(marketmaker_adr)
# we only persist data for xbr markets operated by one of the market makers we run in this worker
if marketmaker_adr_str not in self._maker_adr2id or self._maker_adr2id[
marketmaker_adr_str] not in self._makers:
self.log.info(
'{event}: skipping channel (channel {channel_oid} in market with market maker address {marketmaker_adr} is in for any market in this markets worker)',
event=hlcontract('XBRChannel.Opened'),
channel_oid=hlid(channel_oid),
marketmaker_adr=hlid(marketmaker_adr_str))
return
# prepare new channel data
channel_type = int(args.ctype)
market_oid = uuid.UUID(bytes=args.marketId)
actor_adr = bytes(HexBytes(args.actor))
delegate_adr = bytes(HexBytes(args.delegate))
recipient_adr = bytes(HexBytes(args.recipient))
amount = int(args.amount)
# FIXME
# signature = bytes(HexBytes(args.signature))
# FIXME
# member_oid = uuid.UUID()
# get the market maker by address
maker = self._makers[self._maker_adr2id[marketmaker_adr_str]]
# the market maker specific embedded database and schema
db = maker.db
xbrmm = maker.schema
# depending on channel type, different database schema classes and tables are used
if channel_type == cfxdb.xbrmm.ChannelType.PAYMENT:
channel = cfxdb.xbrmm.PaymentChannel()
channels = xbrmm.payment_channels
channels_by_delegate = xbrmm.idx_payment_channel_by_delegate
balance = cfxdb.xbrmm.PaymentChannelBalance()
balances = xbrmm.payment_balances
elif channel_type == cfxdb.xbrmm.ChannelType.PAYING:
channel = cfxdb.xbrmm.PayingChannel()
channels = xbrmm.paying_channels
channels_by_delegate = xbrmm.idx_paying_channel_by_delegate
balance = cfxdb.xbrmm.PayingChannelBalance()
balances = xbrmm.paying_balances
else:
assert False, 'should not arrive here'
# fill in information for newly replicated channel
channel.market_oid = market_oid
# FIXME
# channel.member_oid = member_oid
channel.channel_oid = channel_oid
channel.timestamp = np.datetime64(time_ns(), 'ns')
# channel.open_at = None
# FIXME: should read that from even args after deployment of
# https://github.com/crossbario/xbr-protocol/pull/138
channel.seq = 1
channel.channel_type = channel_type
channel.marketmaker = marketmaker_adr
channel.actor = actor_adr
channel.delegate = delegate_adr
channel.recipient = recipient_adr
channel.amount = amount
# FIXME
channel.timeout = 0
channel.state = cfxdb.xbrmm.ChannelState.OPEN
# FIXME
# channel.open_sig = signature
# create an off-chain balance record for the channel with remaining == initial amount
balance.remaining = channel.amount
# FIXME: should read that from even args after deployment of
# https://github.com/crossbario/xbr-protocol/pull/138
balance.seq = 1
# now store the new channel and balance in the database
stored = False
cnt_channels_before = 0
cnt_channels_by_delegate_before = 0
cnt_channels_after = 0
cnt_channels_by_delegate_after = 0
with db.begin(write=True) as txn:
if channels[txn, channel_oid]:
self.log.warn('{event}: channel already stored in database [channel_oid={channel_oid}]',
event=hlcontract('XBRChannel.Opened'),
channel_oid=hlid(channel_oid))
else:
cnt_channels_before = channels.count(txn)
cnt_channels_by_delegate_before = channels_by_delegate.count(txn)
# store the channel along with the off-chain balance
channels[txn, channel_oid] = channel
balances[txn, channel_oid] = balance
stored = True
cnt_channels_after = channels.count(txn)
cnt_channels_by_delegate_after = channels_by_delegate.count(txn)
self.log.info(
'{event} DB result: stored={stored}, cnt_channels_before={cnt_channels_before}, cnt_channels_by_delegate_before={cnt_channels_by_delegate_before}, cnt_channels_after={cnt_channels_after}, cnt_channels_by_delegate_after={cnt_channels_by_delegate_after}',
event=hlcontract('XBRChannel.Opened'),
stored=hlval(stored),
cnt_channels_before=hlval(cnt_channels_before),
cnt_channels_by_delegate_before=hlval(cnt_channels_by_delegate_before),
cnt_channels_after=hlval(cnt_channels_after),
cnt_channels_by_delegate_after=hlval(cnt_channels_by_delegate_after))
if stored:
# FIXME: publish WAMP event
self.log.info(
'{event}: new channel stored in database [actor_adr={actor_adr}, channel_type={channel_type}, market_oid={market_oid}, member_oid={member_oid}, channel_oid={channel_oid}]',
event=hlcontract('XBRChannel.Opened'),
market_oid=hlid(market_oid),
# FIXME
member_oid=hlid(None),
channel_oid=hlid(channel_oid),
actor_adr=hlid('0x' + binascii.b2a_hex(actor_adr).decode()),
channel_type=hlid(channel_type))
def load_config(self, configfile=None):
"""
Load the node configuration of CFC itself. The base node configuration
is built into CFC, but can be overridden (partially) and extended
by providing a regular, local node configuration file.
When such a file exists:
- and it contains a controller section, these take precedence over
the builtin values.
- and it contains a workers section (a list of workers), these are
_added_ to the builtin workers.
"""
config_source = Node.CONFIG_SOURCE_EMPTY
# 1/4: load builtin master node configuration as default
#
config_path = pkg_resources.resource_filename('crossbar',
self.DEFAULT_CONFIG_PATH)
with open(config_path) as f:
config = json.load(f)
# no need to check the builtin config (at run-time)
# self.personality.check_config(self.personality, config)
# remember config source
config_source += Node.CONFIG_SOURCE_DEFAULT
self.log.info('Built-in master node configuration loaded')
# 2/4: allow extending/overriding the node configuration
# with a local master node configuration file (eg for TLS setup)
if configfile:
config_path = os.path.abspath(os.path.join(self._cbdir,
configfile))
with open(config_path) as f:
custom_config = json.load(f)
# as an overriding config file does not need to be a fully valid config in itself,
# do _not_ check it ..
# self.personality.check_config(self.personality, custom_config)
# .. instead, we merge the config from the local file into the already
# loaded default config (see above)
config = merge_config(config, custom_config)
# remember config source
config_source += Node.CONFIG_SOURCE_LOCALFILE
self.log.info(
'Local master node configuration merged from "{config_path}"',
config_path=hlval(config_path))
# 3/4: allow extending/overriding the node configuration
# with a remote master node configuration file from blockchain/IPFS
if config.get('controller', {}).get('blockchain', None):
blockchain_config = config['controller']['blockchain']
gateway_config = blockchain_config['gateway']
# setup Web3 connection from blockchain gateway configuration
self._w3 = make_w3(gateway_config)
# configure new Web3 connection as provider for XBR
xbr.setProvider(self._w3)
if self._w3.isConnected():
self.log.info(
'Connected to Ethereum network {network} at gateway "{gateway_url}"',
network=self._w3.net.version,
gateway_url=gateway_config['http'])
# try to find node by node public key on-chain
xbr_node_id = xbr.xbrnetwork.functions.getNodeByKey(
self.key.public_key()).call()
if xbr_node_id != b'\x00' * 16:
assert (len(xbr_node_id) == 16)
# get node domain, type, configuration and license as stored on-chain
xbr_node_domain = xbr.xbrnetwork.functions.getNodeDomain(
xbr_node_id).call()
xbr_node_type = xbr.xbrnetwork.functions.getNodeType(
xbr_node_id).call()
xbr_node_config = xbr.xbrnetwork.functions.getNodeConfig(
xbr_node_id).call()
# FIXME: for testing use hard-coded "trial license" (no. 20010)
xbr_node_license = 20010
# xbr_node_license = xbr.xbrnetwork.functions.getNodeLicense(xbr_node_id).call()
self.log.info(
'Node is configured for XBR (xbr_node_id="{xbr_node_id}", xbr_node_domain="{xbr_node_domain}", xbr_node_type="{xbr_node_type}", xbr_node_license="{xbr_node_license}", xbr_node_config="{xbr_node_config}")',
xbr_node_id=hlid(
'0x' + binascii.b2a_hex(xbr_node_id).decode()),
xbr_node_domain=hlid(
'0x' + binascii.b2a_hex(xbr_node_domain).decode()),
xbr_node_type=hlid(xbr_node_type),
xbr_node_config=hlid(xbr_node_config),
xbr_node_license=hlid(xbr_node_license))
self._license = License(xbr_node_license)
self.log.info(
'Node is registered in the XBR network with license type={license}',
license=self._license.type)
# if a (hash of a) configuration is set on-chain for the master node, fetch the
# configuration content for the hash from IPFS
if xbr_node_config:
if 'IPFS_GATEWAY_URL' in os.environ:
ipfs_gateway_url = os.environ['IPFS_GATEWAY_URL']
self.log.info(
'Using explicit IPFS gateway URL {ipfs_gateway_url} from environment variable IPFS_GATEWAY_URL',
ipfs_gateway_url=hlid(ipfs_gateway_url))
else:
ipfs_gateway_url = 'https://ipfs.infura.io:5001'
self.log.info(
'Using default IPFS Infura gateway URL {ipfs_gateway_url}',
ipfs_gateway_url=hlid(ipfs_gateway_url))
ipfs_config_url = '{}/api/v0/cat?arg={}&encoding=json'.format(
ipfs_gateway_url, xbr_node_config)
resp = requests.get(ipfs_config_url)
xbr_node_config_data = resp.json()
from pprint import pprint
pprint(xbr_node_config_data)
self.personality.check_config(self.personality,
xbr_node_config_data)
if 'controller' not in xbr_node_config_data:
xbr_node_config_data['controller'] = {}
if 'id' not in xbr_node_config_data['controller']:
xbr_node_config_data['controller']['id'] = str(
uuid.UUID(bytes=xbr_node_id))
self.log.info(
'Deriving node ID {node_id} from XBR network node ID',
node_id=hlid(
xbr_node_config_data['controller']['id']))
else:
self.log.info(
'Setting node ID {node_id} from XBR network node configuration',
node_id=hlid(
xbr_node_config_data['controller']['id']))
self._config = xbr_node_config_data
self.log.info(
'Node configuration loaded from XBR network (xbr_node_id="{xbr_node_id}")',
xbr_node_id=hlid(
'0x' + binascii.b2a_hex(xbr_node_id).decode()))
else:
self.log.debug(
'There is no node configuration stored in XBR network (xbr_node_id="{xbr_node_id}")',
xbr_node_id=hlid(
'0x' + binascii.b2a_hex(xbr_node_id).decode()))
config_source = self.CONFIG_SOURCE_XBRNETWORK
else:
self.log.warn(
'Could not find node public key on blockchain yet')
else:
self.log.warn('Could not connect to Ethereum blockchain')
# 4/4: check and set the final merged master node configuration
#
self.personality.check_config(self.personality, config)
self._config = config
self.log.debug(
'Master node config after (effective after merge):\n{config}',
config=pformat(config))
return config_source, config_path
