def main(reactor):
# create an etcd client
etcd = Client(reactor)
# retrieve etcd cluster status
status = yield etcd.status()
print(status)
# callback invoked for every change
def on_change(kv):
print('on_change: {}'.format(kv))
# start watching on given keys or key sets
# when the key sets overlap, the callback might get called multiple
# times, once for each key set matching an event
keys = [KeySet(b'mykey2', b'mykey5'), KeySet(b'mykey', prefix=True)]
d = etcd.watch(keys, on_change)
# stop after 10 seconds
print('watching for 1s ..')
yield txaio.sleep(1)
etcd.set(b'mykey10', b'Test')
print('watching for 3s ..')
yield txaio.sleep(3)
d.cancel()
def transport_check(_):
self.log.debug('Entering re-connect loop')
if not self._can_reconnect():
err_msg = u"Component failed: Exhausted all transport connect attempts"
self.log.info(err_msg)
try:
raise RuntimeError(err_msg)
except RuntimeError as e:
txaio.reject(self._done_f, e)
return
while True:
transport = next(transport_gen)
if transport.can_reconnect():
transport_candidate[0] = transport
break
delay = transport.next_delay()
self.log.debug(
'trying transport {transport_idx} using connect delay {transport_delay}',
transport_idx=transport.idx,
transport_delay=delay,
)
self._delay_f = txaio.sleep(delay)
txaio.add_callbacks(self._delay_f, attempt_connect, error)
def transport_check(_):
self.log.debug('Entering re-connect loop')
if not self._can_reconnect():
err_msg = u"Component failed: Exhausted all transport connect attempts"
self.log.info(err_msg)
try:
raise RuntimeError(err_msg)
except RuntimeError as e:
txaio.reject(self._done_f, e)
return
while True:
transport = next(transport_gen)
if transport.can_reconnect():
transport_candidate[0] = transport
break
delay = transport.next_delay()
self.log.debug(
'trying transport {transport_idx} using connect delay {transport_delay}',
transport_idx=transport.idx,
transport_delay=delay,
)
self._delay_f = txaio.sleep(delay)
txaio.add_callbacks(self._delay_f, attempt_connect, error)
def stop(self):
"""
Stop the (currently running) Web-cluster manager.
:return:
"""
assert self._started > 0, 'cannot stop Metadata manager - currently not running!'
yield sleep(0)
self._started = None
def example2(reactor, etcd):
print("\n\nEXAMPLE 2")
print('creating lease with 5s TTL')
lease = yield etcd.lease(5)
print(lease)
print('refreshing lease every 4s, 5 times ..')
for i in range(5):
rev = yield lease.refresh()
print(rev)
yield txaio.sleep(4)
print('sleeping for 6s ..')
yield txaio.sleep(6)
print('refreshing lease')
try:
yield lease.refresh()
except Expired:
print('leave expired (expected)')
def example3(reactor, etcd):
print("\n\nEXAMPLE 3")
print('creating lease with 5s TTL')
lease = yield etcd.lease(5)
print(lease)
print('sleeping for 2s ..')
yield txaio.sleep(2)
print('revoking lease')
res = yield lease.revoke()
print(res)
print('refreshing lease')
try:
yield lease.refresh()
except Expired:
print('leave expired (expected)')
def example1(reactor, etcd):
print("\n\nEXAMPLE 1")
print('creating lease with 5s TTL')
lease = yield etcd.lease(5)
print(lease)
i = 1
while True:
try:
remaining = yield lease.remaining()
i += 1
except Expired:
if i == 5:
print('lease expired (expected)')
break
else:
raise
else:
print('lease TTL = {}'.format(remaining))
print('sleeping for 1s ..')
yield txaio.sleep(1)
def example4(reactor, etcd):
print("\n\nEXAMPLE 4")
print('creating lease with 5s TTL')
lease = yield etcd.lease(5)
print(lease)
yield etcd.set(b'foo', b'bar', lease=lease)
i = 0
while True:
result = yield etcd.get(b'foo')
if result.kvs:
kv = result.kvs[0]
print(kv)
i += 1
else:
print('key has been deleted together with expired lease ({}s)'.
format(i))
break
print('sleeping for 1s ..')
yield txaio.sleep(1)
def one_reconnect_loop(_):
self.log.debug('Entering re-connect loop')
if not reconnect[0]:
return
# cycle through all transports forever ..
transport = next(transport_gen)
# only actually try to connect using the transport,
# if the transport hasn't reached max. connect count
if transport.can_reconnect():
delay = transport.next_delay()
self.log.debug(
'trying transport {transport_idx} using connect delay {transport_delay}',
transport_idx=transport.idx,
transport_delay=delay,
)
delay_f = asyncio.ensure_future(txaio.sleep(delay))
def actual_connect(_):
f = self._connect_once(loop, transport)
def session_done(x):
txaio.resolve(done_f, None)
def connect_error(fail):
if isinstance(fail.value, asyncio.CancelledError):
reconnect[0] = False
txaio.reject(done_f, fail)
return
self.log.debug(u'component failed: {error}',
error=txaio.failure_message(fail))
self.log.debug(u'{tb}',
tb=txaio.failure_format_traceback(fail))
# If this is a "fatal error" that will never work,
# we bail out now
if isinstance(fail.value, ApplicationError):
if fail.value.error in [
u'wamp.error.no_such_realm'
]:
reconnect[0] = False
self.log.error(
u"Fatal error, not reconnecting")
txaio.reject(done_f, fail)
return
self.log.error(u"{msg}",
msg=fail.value.error_message())
return one_reconnect_loop(None)
elif isinstance(fail.value, OSError):
# failed to connect entirely, like nobody
# listening etc.
self.log.info(u"Connection failed: {msg}",
msg=txaio.failure_message(fail))
return one_reconnect_loop(None)
elif _is_ssl_error(fail.value):
# Quoting pyOpenSSL docs: "Whenever
# [SSL.Error] is raised directly, it has a
# list of error messages from the OpenSSL
# error queue, where each item is a tuple
# (lib, function, reason). Here lib, function
# and reason are all strings, describing where
# and what the problem is. See err(3) for more
# information."
self.log.error(u"TLS failure: {reason}",
reason=fail.value.args[1])
self.log.error(u"Marking this transport as failed")
transport.failed()
else:
self.log.error(
u'Connection failed: {error}',
error=txaio.failure_message(fail),
)
# This is some unknown failure, e.g. could
# be SyntaxError etc so we're aborting the
# whole mission
txaio.reject(done_f, fail)
return
txaio.add_callbacks(f, session_done, connect_error)
txaio.add_callbacks(delay_f, actual_connect, error)
def twisted_main(reactor,
stdscr=None,
mrealms=None,
management_url=None,
privkey_file=None):
mrealms = mrealms or ['default']
try:
if stdscr:
stdscr.clear()
y = 5
for line in personality.Personality.BANNER.splitlines():
stdscr.addstr(y, 20, line, curses.color_pair(227))
y += 1
y += 3
stdscr.addstr(
y, 24,
'Please wait while collecting data from managed nodes ...')
stdscr.refresh()
while True:
if stdscr:
stdscr.clear()
y = 0
stdscr.addstr(y, 0, '=' * 240)
y += 1
x = 0
stdscr.addstr(y, x, 'Node')
x += 34
stdscr.addstr(y, x, 'Mgmt Realm', curses.color_pair(14))
x += 15
stdscr.addstr(y, x, 'Node ID', curses.color_pair(14))
x += 25
stdscr.addstr(y, x, 'Node OID', curses.color_pair(14))
x += 40
stdscr.addstr(y, x, 'Status')
x += 10
stdscr.addstr(y, x, 'Last Heartbeat')
x += 12
x += 4
stdscr.addstr(y, x + 0, ' Usr')
stdscr.addstr(y, x + 5, ' Sys')
stdscr.addstr(y, x + 10, ' Idl')
x += 1
stdscr.addstr(y, x + 15, ' Mem')
stdscr.addstr(y, x + 20, ' IPv4 sckts')
x += 5 * 5
x += 11
stdscr.addstr(y, x + 0, ' Pxy', curses.color_pair(41))
stdscr.addstr(y, x + 4, ' Rtr', curses.color_pair(41))
stdscr.addstr(y, x + 8, ' Xbr', curses.color_pair(41))
stdscr.addstr(y, x + 12, ' Cnt', curses.color_pair(41))
stdscr.addstr(y, x + 16, ' Gst', curses.color_pair(41))
x += 4 * 5
x += 4
stdscr.addstr(y, x + 0, ' Rlm', curses.color_pair(227))
stdscr.addstr(y, x + 5, ' Rls')
stdscr.addstr(y, x + 10, ' Rlk')
stdscr.addstr(y, x + 15, ' Sessions', curses.color_pair(41))
stdscr.addstr(y, x + 25, ' Messages', curses.color_pair(41))
x += 5 * 5
y += 1
stdscr.addstr(y, 0, '-' * 240)
y += 1
last_mrealm = None
for mrealm in mrealms:
if stdscr:
if last_mrealm:
stdscr.addstr(y, 0, '.' * 240)
y += 1
try:
session, _ = yield create_management_session(
url=management_url,
realm=mrealm,
privkey_file=privkey_file)
except Exception as e:
print(e)
node_oids = yield session.call(
'crossbarfabriccenter.mrealm.get_nodes')
if not stdscr:
pprint(node_oids)
for node_oid in node_oids:
node = yield session.call(
'crossbarfabriccenter.mrealm.get_node', node_oid)
if not stdscr:
pprint(node)
if True:
node_authid = node['authid']
# node_pubkey = node['pubkey']
if node and node['status'] == 'online':
node_status = yield session.call(
'crossbarfabriccenter.remote.node.get_status',
node_oid)
if not stdscr:
pprint(node_status)
node_system_stats = yield session.call(
'crossbarfabriccenter.remote.node.get_system_stats',
node_oid)
if not stdscr:
pprint(node_system_stats)
cpu_user = node_system_stats['cpu']['user']
cpu_system = node_system_stats['cpu']['system']
cpu_idle = node_system_stats['cpu']['idle']
# memory_total = node_system_stats['memory']['total']
# memory_avail = node_system_stats['memory']['available']
memory_perc = node_system_stats['memory'][
'percent']
# network_recv = node_system_stats['network']['bytes_recv_per_sec']
# network_sent = node_system_stats['network']['bytes_sent_per_sec']
network_conns = node_system_stats['network'][
'connection']['AF_INET']
# tz = get_timezone('Europe/Berlin')
# started = format_datetime(iso8601.parse_date(node_status['started']), tzinfo=tz, locale='de_DE', format='long')
last_heartbeat = np.datetime64(
node['timestamp'], 'ns')
now = np.datetime64(time_ns(), 'ns')
if now > last_heartbeat:
last_heartbeat_ago = str(
(now -
last_heartbeat).astype("timedelta64[s]"))
else:
last_heartbeat_ago = None
node_title = node_status['title']
# get IDs for all workers running in this node
worker_info = {}
router_info = {}
workers = yield session.call(
'crossbarfabriccenter.remote.node.get_workers',
node_oid)
for worker_id in workers:
# get worker detail information
# {'id': 'xbr1', 'pid': 11507, 'type': 'marketplace', 'status': 'started', 'created': '2020-06-22T06:15:44.589Z', 'started': '2020-06-22T06:15:48.224Z', 'startup_time': 3.635574, 'uptime': 13949.814363}
worker = yield session.call(
'crossbarfabriccenter.remote.node.get_worker',
node_oid, worker_id)
if not stdscr:
pprint(worker)
if worker['status'] == 'started':
if worker['type'] not in worker_info:
worker_info[worker['type']] = 0
worker_info[worker['type']] += 1
if worker['type'] == 'router':
# get IDs for all realm running in router worker
realm_oids = yield session.call(
'crossbarfabriccenter.remote.router.get_router_realms',
node_oid, worker_id)
for realm_oid in realm_oids:
# get realm detail information
realm = yield session.call(
'crossbarfabriccenter.remote.router.get_router_realm',
node_oid, worker_id, realm_oid)
if not stdscr:
pprint(realm)
# get per-realm messaging statistics
realm_stats = yield session.call(
'crossbarfabriccenter.remote.router.get_router_realm_stats',
node_oid, worker_id, realm_oid)
if not stdscr:
pprint(realm_stats)
realm_id = realm['id']
realm_name = realm['config'][
'name']
realm_created = realm['created']
ri_obj = {
'node_oid':
node_oid,
'worker_id':
worker_id,
'id':
realm_id,
'name':
realm_name,
'created':
realm_created,
'rlinks':
len([
1 for rlink in
realm['rlinks']
if rlink['connected']
]),
}
sw_latest = '20.6.2.dev2' in node_status[
'title']
if sw_latest:
# get IDs of all rlinks running in this router worker and realm
rlink_oids = yield session.call(
'crossbarfabriccenter.remote.router.get_router_realm_links',
node_oid, worker_id,
realm_oid)
ri_obj['rlinks'] = len(
rlink_oids)
# {'realm001': {'messages': {'received': {'publish': 39, 'register': 42},
# 'sent': {'registered': 42}},
# 'roles': 4,
# 'sessions': 2}}
received = realm_stats[realm_id][
'messages']['received']
sent = realm_stats[realm_id][
'messages']['sent']
total = 0
for k in received:
total += received[k]
for k in sent:
total += sent[k]
ri_obj['messages'] = total
ri_obj['received'] = realm_stats[
realm_id]['messages'][
'received']
ri_obj['sent'] = realm_stats[
realm_id]['messages']['sent']
ri_obj['sessions'] = realm_stats[
realm_id]['sessions']
ri_obj['roles'] = realm_stats[
realm_id]['roles']
if realm_name not in router_info:
router_info[realm_name] = [
ri_obj
]
else:
router_info[realm_name].append(
ri_obj)
else:
worker_info = {}
router_info = {}
# started = '-'
last_heartbeat_ago = '-'
node_title = '-'
cpu_user = 0
cpu_system = 0
cpu_idle = 0
# memory_total = 0
# memory_avail = 0
memory_perc = 0
# network_recv = 0
# network_sent = 0
network_conns = 0
if stdscr:
def fmt(data, key):
val = data.get(key, 0)
if val:
return '{0: >4}'.format(
val), curses.color_pair(41)
else:
return ' -', curses.color_pair(8)
x = 0
stdscr.addstr(y, x, node_title)
x += 34
stdscr.addstr(y, x, mrealm, curses.color_pair(14))
x += 15
stdscr.addstr(y, x, node_authid,
curses.color_pair(14))
x += 25
stdscr.addstr(y, x, node_oid,
curses.color_pair(14))
x += 40
if node['status'] == 'online':
stdscr.addstr(y, x, node['status'],
curses.color_pair(41))
else:
stdscr.addstr(y, x, node['status'],
curses.color_pair(10))
x += 10
stdscr.addstr(y, x, last_heartbeat_ago)
x += 12
def fmt2(val):
return '{0: >4}'.format(
val), curses.color_pair(8)
x += 4
stdscr.addstr(y, x + 0, *fmt2(round(cpu_user, 1)))
stdscr.addstr(y, x + 5, *fmt2(round(cpu_system,
1)))
stdscr.addstr(y, x + 10, *fmt2(round(cpu_idle, 1)))
x += 1
stdscr.addstr(y, x + 15,
*fmt2(round(memory_perc, 1)))
x += 1
stdscr.addstr(y, x + 20,
'{0: >10}'.format(network_conns))
x += 5 * 5
x += 10
stdscr.addstr(y, x + 0, *fmt(worker_info, 'proxy'))
stdscr.addstr(y, x + 4,
*fmt(worker_info, 'router'))
stdscr.addstr(y, x + 8,
*fmt(worker_info, 'marketplace'))
stdscr.addstr(y, x + 12,
*fmt(worker_info, 'container'))
stdscr.addstr(y, x + 16,
*fmt(worker_info, 'guest'))
x += 4 * 5
# cpu_user = node_system_stats['cpu']['user']
# cpu_system = node_system_stats['cpu']['system']
# cpu_idle = node_system_stats['cpu']['idle']
# memory_total = node_system_stats['memory']['total']
# memory_avail = node_system_stats['memory']['available']
# memory_perc = node_system_stats['memory']['percent']
# network_recv = node_system_stats['network']['bytes_recv_per_sec']
# network_sent = node_system_stats['network']['bytes_sent_per_sec']
# network_conns = node_system_stats['network']['connection']['AF_INET']
roles = 0
sessions = 0
messages = 0
rlinks = 0
for realm_id in router_info:
for realm_obj in router_info[realm_id]:
roles += realm_obj['roles']
sessions += realm_obj['sessions']
messages += realm_obj['messages']
rlinks += realm_obj['rlinks']
if stdscr:
x += 4
stdscr.addstr(
y, x + 0,
'{0: >4}'.format(len(router_info.keys())),
curses.color_pair(227))
stdscr.addstr(y, x + 5, '{0: >4}'.format(roles))
stdscr.addstr(y, x + 10, '{0: >4}'.format(rlinks))
stdscr.addstr(y, x + 15,
'{0: >10}'.format(sessions),
curses.color_pair(41))
stdscr.addstr(y, x + 25,
'{0: >10}'.format(messages),
curses.color_pair(41))
x += 5 * 5
y += 1
last_mrealm = mrealm
if stdscr:
stdscr.addstr(y, 0, '=' * 240)
y += 1
stdscr.refresh()
yield sleep(5)
except Exception as e:
sys.stderr.write(str(e))
sys.exit(1)
def onJoin(self, details):
# self.log.info("{obj_id} connected: {details}", obj_id=id(self), details=details)
bar = {
'realm': details.realm,
'session': details.session,
'authid': details.authid,
'authrole': details.authrole,
'authmethod': details.authmethod,
'authprovider': details.authprovider,
'authextra': details.authextra,
'serializer': details.serializer,
'transport': details.transport,
'node': details.authextra.get('x_cb_node', None),
'worker': details.authextra.get('x_cb_worker', None),
'pid': details.authextra.get('x_cb_pid', None),
'proxy_node': details.authextra.get('x_cb_proxy_node', None),
'proxy_worker': details.authextra.get('x_cb_proxy_worker', None),
'x_cb_proxy_pid': details.authextra.get('x_cb_proxy_pid', None),
}
self._wamp_sessions[id(self)] = bar
self._wamp_log.append(
(id(self), 'JOIN', time_ns(), self.session_id, bar))
# print(id(self), 'JOIN')
mytopic1 = u"com.example.mytopic1"
def on_mytopic1(*args, **kwargs):
details = kwargs.pop('details', None)
assert 'foo' in kwargs and type(kwargs['foo']) == str and len(
kwargs['foo']) == 22
assert 'bar' in kwargs and type(kwargs['bar']) == dict
self._wamp_log.append(
(id(self), 'EVENT', time_ns(), self.session_id, mytopic1, args,
kwargs, details.publication if details else None))
# print(id(self), 'EVENT')
sub = yield self.subscribe(on_mytopic1, mytopic1)
self._running = True
ready1 = self.config.extra.get('ready1', None)
if ready1 and not ready1.called:
ready1.callback((self, bar))
continue1 = self.config.extra.get('continue1', None)
if continue1:
yield continue1
pid = os.getpid()
counter = 0
print('starting loop on {} for {} repeats ..'.format(
id(self), self._repeat))
while self.is_connected() and counter < self._repeat:
# print("pid {} publish {} to '{}'".format(pid, counter, mytopic1))
baz = os.urandom(10)
args = [pid, counter]
kwargs = {'foo': '0x' + binascii.b2a_hex(baz).decode(), 'bar': bar}
pub = yield self.publish(
mytopic1,
*args,
**kwargs,
options=PublishOptions(acknowledge=True, exclude_me=False),
)
self._wamp_log.append(
(id(self), 'PUBLISH', time_ns(), self.session_id, mytopic1,
args, kwargs, pub.id if pub else None))
# print(id(self), 'PUBLISH')
counter += 1
yield sleep(.1)
ready2 = self.config.extra.get('ready2', None)
if ready2 and not ready2.called:
ready2.callback((self, bar))
continue2 = self.config.extra.get('continue2', None)
if continue2:
yield continue2
yield sub.unsubscribe()
self.leave()
def one_reconnect_loop(_):
self.log.debug('Entering re-connect loop')
if not reconnect[0]:
return
# cycle through all transports forever ..
transport = next(transport_gen)
# only actually try to connect using the transport,
# if the transport hasn't reached max. connect count
if transport.can_reconnect():
delay = transport.next_delay()
self.log.debug(
'trying transport {transport_idx} using connect delay {transport_delay}',
transport_idx=transport.idx,
transport_delay=delay,
)
delay_f = asyncio.ensure_future(txaio.sleep(delay))
def actual_connect(_):
f = self._connect_once(loop, transport)
def session_done(x):
txaio.resolve(done_f, None)
def connect_error(fail):
if isinstance(fail.value, asyncio.CancelledError):
reconnect[0] = False
txaio.reject(done_f, fail)
return
self.log.debug(u'component failed: {error}', error=txaio.failure_message(fail))
self.log.debug(u'{tb}', tb=txaio.failure_format_traceback(fail))
# If this is a "fatal error" that will never work,
# we bail out now
if isinstance(fail.value, ApplicationError):
if fail.value.error in [u'wamp.error.no_such_realm']:
reconnect[0] = False
self.log.error(u"Fatal error, not reconnecting")
txaio.reject(done_f, fail)
return
self.log.error(u"{msg}", msg=fail.value.error_message())
return one_reconnect_loop(None)
elif isinstance(fail.value, OSError):
# failed to connect entirely, like nobody
# listening etc.
self.log.info(u"Connection failed: {msg}", msg=txaio.failure_message(fail))
return one_reconnect_loop(None)
elif _is_ssl_error(fail.value):
# Quoting pyOpenSSL docs: "Whenever
# [SSL.Error] is raised directly, it has a
# list of error messages from the OpenSSL
# error queue, where each item is a tuple
# (lib, function, reason). Here lib, function
# and reason are all strings, describing where
# and what the problem is. See err(3) for more
# information."
self.log.error(u"TLS failure: {reason}", reason=fail.value.args[1])
self.log.error(u"Marking this transport as failed")
transport.failed()
else:
self.log.error(
u'Connection failed: {error}',
error=txaio.failure_message(fail),
)
# some types of errors should probably have
# stacktraces logged immediately at error
# level, e.g. SyntaxError?
self.log.debug(u'{tb}', tb=txaio.failure_format_traceback(fail))
return one_reconnect_loop(None)
txaio.add_callbacks(f, session_done, connect_error)
txaio.add_callbacks(delay_f, actual_connect, error)
if False:
# check if there is any transport left we can use
# to connect
if not self._can_reconnect():
self.log.info("No remaining transports to try")
reconnect[0] = False
def main(reactor):
if True:
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = MySchema()
with zlmdb.Database(dbpath) as db:
# write records into zlmdb
with db.begin(write=True) as txn:
for i in range(10):
key = 'key{}'.format(i)
value = 'value{}'.format(random.randint(0, 1000))
schema.samples[txn, key] = value
# read records from zlmdb
with db.begin() as txn:
for i in range(10):
key = 'key{}'.format(i)
value = schema.samples[txn, key]
print('key={} : value={}'.format(key, value))
if True:
# etcd database
etcd = Client(reactor)
status = yield etcd.status()
print(status)
# zlmdb database
schema = MySchema()
dbpath = '/tmp/.test-zlmdb'
with zlmdb.Database(dbpath) as db:
print('zlmdb open on {}'.format(dbpath))
# check current record count
with db.begin() as txn:
cnt = schema.samples.count(txn)
print('currently {} rows in table'.format(cnt))
# watch changes in etcd and write to local zlmdb
def on_change(kv):
key = kv.key.decode()
value = kv.value.decode()
with db.begin(write=True) as txn:
schema.samples[txn, key] = value
print(
'on_change received from etcd and written to zlmdb: key={} value={}'
.format(key, value))
# start watching for etcd changes ..
ks = [KeySet('k'.encode(), prefix=True)]
d = etcd.watch(ks, on_change)
print('watching for 1s ..')
yield txaio.sleep(1)
# loop every 1s and write a key-value in etcd directly
for i in range(5):
print('watching for 1s ..')
yield txaio.sleep(1)
key = 'key{}'.format(i).encode()
value = 'value{}'.format(random.randint(0, 1000)).encode()
etcd.set(key, value)
# cancel our watch
d.cancel()
yield util.sleep(1)
# check current record count
with db.begin() as txn:
cnt = schema.samples.count(txn)
print('currently {} rows in table'.format(cnt))
yield util.sleep(1)
