def get_state(breadcrumbs, k):
'''
build a state request message, send and return the result
breadcrumbs - breadcrumbs dict
k - key for pertinent breadcrumb
'''
msgTx = bc_common.build_msg(breadcrumbs, k)
msgTx.state.Clear()
bc_common.send_msg(breadcrumbs, k, msgTx)
msgRx = bc_common.recv_msg(breadcrumbs, k)
return msgRx
def get_file(breadcrumbs, k, task_id):
'''
build a file download request message, send and return the result
breadcrumbs - breadcrumbs dict
k - key for pertinent breadcrumb
'''
msgTx = bc_common.build_msg(breadcrumbs, k)
msgTx.taskOutputRequest.position = 0
msgTx.taskOutputRequest.maximumDataSize = 65535
msgTx.taskOutputRequest.id = task_id
bc_common.send_msg(breadcrumbs, k, msgTx)
msgRx = bc_common.recv_msg(breadcrumbs, k)
return msgRx
def set_name(breadcrumbs, k, name):
'''
change the name of a breadcrumb
breadcrumbs - breadcrumbs dict
k - key for pertinent breadcrumb
name - new name for breadcrumb
'''
msgTx = bc_common.build_msg(breadcrumbs, k)
msgTx.config.general.name = name
bc_common.send_msg(breadcrumbs, k, msgTx)
msgRx = bc_common.recv_msg(breadcrumbs, k)
return msgRx
def set_watch(breadcrumbs, k, watches, interval=5):
'''
build a watch request message, send and return the result
breadcrumbs - breadcrumbs dict
k - key for pertinent breadcrumb
watches - list of BCMessage paths to watch (assuming state. prefix)
interval - interval to ask for them to be watch
'''
msgTx = bc_common.build_msg(breadcrumbs, k)
for w in watches:
watchObj = msgTx.watchRequest.watchObject.add()
watchObj.messagePath = w
watchObj.interval = interval
bc_common.send_msg(breadcrumbs, k, msgTx)
msgRx = bc_common.recv_msg(breadcrumbs, k)
return msgRx
def get_trace(breadcrumbs, k, ip):
'''
build a trace request message, send and return the result
breadcrumbs - breadcrumbs dict
k - key for pertinent breadcrumb
'''
msgTx = bc_common.build_msg(breadcrumbs, k)
tasks = {
k: v for
k, v in
Message_pb2.Common_pb2.TaskCommand.TaskAction.items()
}
msgTx.runTask.action = tasks['TRACE']
msgTx.runTask.arguments = ip
bc_common.send_msg(breadcrumbs, k, msgTx)
msgRx = bc_common.recv_msg(breadcrumbs, k)
return msgRx
def reboot(breadcrumbs, k):
'''
reboot a breadcrumb
breadcrumbs - breadcrumbs dict
k - key for pertinent breadcrumb
'''
msgTx = bc_common.build_msg(breadcrumbs, k)
tasks = {
k: v for
k, v in
Message_pb2.Common_pb2.TaskCommand.TaskAction.items()
}
msgTx.runTask.action = tasks['REBOOT']
bc_common.send_msg(breadcrumbs, k, msgTx)
msgRx = bc_common.recv_msg(breadcrumbs, k)
return msgRx
def gatherer(breadcrumbs, k, trace_ip):
'''
designed to be spawned as a thread; loops around and gathers all the BC
stats
breadcrumbs - breadcrumbs dict
k - key for pertinent breadcrumb
trace_ip - ip address of trace target
'''
pnow = datetime.datetime.now()
print pnow, 'gatherer - started', k
first_run = True
stat = {
'timestamp': None,
'ip': '',
'name': '',
'wireless': {},
'wired': {},
'nexthop_mac': '',
'nexthop_via': ''
}
while 1:
try:
bc_common.connect(
breadcrumbs,
k,
role='ADMIN',
password='******'
)
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - connect failed', k
time.sleep(5)
continue
while 1:
# build stat structure
now = datetime.datetime.now()
stat['timestamp'] = now
# get complete breadcrumb state first time, set watch for updates
if first_run:
# get full state
try:
response = bc_query.get_state(breadcrumbs, k)
first_run = False
except:
traceback.print_exc()
pnow = datetime.datetime.now()
print pnow, 'gatherer - get_state failed', k
break
# setup watch
try:
bc_query.set_watch(
breadcrumbs,
k,
[
'system.ipv4',
'wired',
'wireless',
'task',
'configuration.saved.general.name'
],
8 # watch response interval
)
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - set_watch failed', k
break
else:
# block until watch response received
try:
response = bc_common.recv_msg(breadcrumbs, k).watchResponse
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - recv_msg failed', k
break
# get ip
stat['ip'] = breadcrumbs[k]['ip']
# get name
name = response.state.configuration.saved.general.name
if name != '':
stat['name'] = name
# get wired data
for wv in response.state.wired:
try:
wmac = ':'.join([
str(hex(x)).split('x')[1].zfill(2)
for x in struct.unpack("BBBBBB", wv.mac)
])
except:
continue
if wmac not in stat['wired']:
stat['wired'][wmac] = {}
stat['wired'][wmac].update({
'name': wv.name,
'rx_bytes': wv.stats.rxBytes,
'tx_bytes': wv.stats.txBytes,
})
# get wireless data
for wv in response.state.wireless:
try:
wmac = ':'.join([
str(hex(x)).split('x')[1].zfill(2)
for x in struct.unpack("BBBBBB", wv.mac)
])
except:
continue
if wmac not in stat['wireless']:
stat['wireless'][wmac] = {'peers': {}}
stat['wireless'][wmac].update({
'name': wv.name,
'noise': wv.noise,
'channel': wv.channel,
'txpower': wv.txpower,
'tx_bytes': wv.stats.txBytes,
'rx_bytes': wv.stats.rxBytes,
'peers': {}
})
# get peer data
for pv in wv.peer:
try:
pmac = ':'.join([
str(hex(x)).split('x')[1].zfill(2)
for x in struct.unpack("BBBBBB", pv.mac)
])
except:
continue
if pmac not in stat['wireless'][wmac]['peers']:
stat['wireless'][wmac]['peers'][pmac] = {}
stat['wireless'][wmac]['peers'][pmac].update({
'cost': pv.cost,
'rate': pv.rate / 10,
'rssi': pv.signal
})
# make sure trace succeeded
need_to_trace = False
tasks = {
v: k for k, v in
Common_pb2.TaskCommand.TaskAction.items()
}
statuses = {
v: k for k, v in
Common_pb2.TaskStatus.TaskState.items()
}
if tasks[response.state.task.command.action] != 'TRACE':
pnow = datetime.datetime.now()
print pnow, 'gatherer - last task not trace', k, '(%s)' % (
tasks[response.state.task.command.action]
)
need_to_trace = True
elif statuses[response.state.task.status.state] in [
'DELAYED',
'QUEUED',
'RUNNING'
]:
pnow = datetime.datetime.now()
print pnow, 'gatherer - trace still underway', k, '(%s)' % (
statuses[response.state.task.status.state]
)
need_to_trace = False
elif statuses[response.state.task.status.state] == 'FAILED':
pnow = datetime.datetime.now()
print pnow, 'gatherer - trace failed', k, '(%s)' % (
statuses[response.state.task.status.state]
)
need_to_trace = True
elif statuses[response.state.task.status.state] == 'SUCCESS':
# go and get the gzip'd trace dump
try:
response = bc_query.get_file(
breadcrumbs,
k,
response.state.task.status.id
)
except:
break
statuses = {
v: k for k, v in
Common_pb2.TaskOutputResponse.Status.items()
}
if statuses[response.taskOutputResponse.status] != 'SUCCESS':
pnow = datetime.datetime.now()
print pnow, 'download failed'
continue
# decompress the trace dump
zfile = StringIO.StringIO()
zfile.write(response.taskOutputResponse.data)
zfile.seek(0)
try:
data = gzip.GzipFile(
fileobj=zfile, mode='rb'
).read().strip()
except:
traceback.print_exc()
pnow = datetime.datetime.now()
print pnow, 'gunzip failed'
continue
# parse the trace dump
lines = data.split('\n')[2:]
if len(lines) > 0:
if 'peer=' in lines[0]:
stat['nexthop_via'] = 'mesh' # mesh/failed backhaul
stat['nexthop_mac'] = lines[0].split(
'peer='
)[1].split('/')[0].lower()
else:
stat['nexthop_via'] = 'wire' # ingress/has backhaul
stat['nexthop_mac'] = lines[0].split()[2].lower()
need_to_trace = True
if need_to_trace:
# request a trace task (only returns status of request)
try:
response = bc_query.get_trace(breadcrumbs, k, trace_ip)
time.sleep(5)
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - get_trace failed', k
break
if 'stats' not in breadcrumbs[k]:
breadcrumbs[k]['stats'] = []
breadcrumbs[k]['stats'] += [stat]
breadcrumbs[k]['stats'] = breadcrumbs[k]['stats'][-2:]
pnow = datetime.datetime.now()
print pnow, 'gatherer - got stats for', k
snooze = 8 - (datetime.datetime.now() - now).total_seconds()
time.sleep(snooze if snooze >= 0 else 0)
try:
bc_common.disconnect(
breadcrumbs,
k
)
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - connect failed', k
time.sleep(5)
continue
time.sleep(4)
def gatherer(breadcrumbs, k, trace_ip):
'''
designed to be spawned as a thread; loops around and gathers all the BC
stats
breadcrumbs - breadcrumbs dict
k - key for pertinent breadcrumb
trace_ip - ip address of trace target
'''
pnow = datetime.datetime.now()
print pnow, 'gatherer - started', k
first_run = True
stat = {
'timestamp': None,
'ip': '',
'name': '',
'wireless': {},
'wired': {},
'nexthop_mac': '',
'nexthop_via': ''
}
while 1:
try:
bc_common.connect(breadcrumbs,
k,
role='ADMIN',
password='******')
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - connect failed', k
time.sleep(5)
continue
while 1:
# build stat structure
now = datetime.datetime.now()
stat['timestamp'] = now
# get complete breadcrumb state first time, set watch for updates
if first_run:
# get full state
try:
response = bc_query.get_state(breadcrumbs, k)
first_run = False
except:
traceback.print_exc()
pnow = datetime.datetime.now()
print pnow, 'gatherer - get_state failed', k
break
# setup watch
try:
bc_query.set_watch(
breadcrumbs,
k,
[
'system.ipv4', 'wired', 'wireless', 'task',
'configuration.saved.general.name'
],
8 # watch response interval
)
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - set_watch failed', k
break
else:
# block until watch response received
try:
response = bc_common.recv_msg(breadcrumbs, k).watchResponse
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - recv_msg failed', k
break
# get ip
stat['ip'] = breadcrumbs[k]['ip']
# get name
name = response.state.configuration.saved.general.name
if name != '':
stat['name'] = name
# get wired data
for wv in response.state.wired:
try:
wmac = ':'.join([
str(hex(x)).split('x')[1].zfill(2)
for x in struct.unpack("BBBBBB", wv.mac)
])
except:
continue
if wmac not in stat['wired']:
stat['wired'][wmac] = {}
stat['wired'][wmac].update({
'name': wv.name,
'rx_bytes': wv.stats.rxBytes,
'tx_bytes': wv.stats.txBytes,
})
# get wireless data
for wv in response.state.wireless:
try:
wmac = ':'.join([
str(hex(x)).split('x')[1].zfill(2)
for x in struct.unpack("BBBBBB", wv.mac)
])
except:
continue
if wmac not in stat['wireless']:
stat['wireless'][wmac] = {'peers': {}}
stat['wireless'][wmac].update({
'name': wv.name,
'noise': wv.noise,
'channel': wv.channel,
'txpower': wv.txpower,
'tx_bytes': wv.stats.txBytes,
'rx_bytes': wv.stats.rxBytes,
'peers': {}
})
# get peer data
for pv in wv.peer:
try:
pmac = ':'.join([
str(hex(x)).split('x')[1].zfill(2)
for x in struct.unpack("BBBBBB", pv.mac)
])
except:
continue
if pmac not in stat['wireless'][wmac]['peers']:
stat['wireless'][wmac]['peers'][pmac] = {}
stat['wireless'][wmac]['peers'][pmac].update({
'cost':
pv.cost,
'rate':
pv.rate / 10,
'rssi':
pv.signal
})
# make sure trace succeeded
need_to_trace = False
tasks = {
v: k
for k, v in Common_pb2.TaskCommand.TaskAction.items()
}
statuses = {
v: k
for k, v in Common_pb2.TaskStatus.TaskState.items()
}
if tasks[response.state.task.command.action] != 'TRACE':
pnow = datetime.datetime.now()
print pnow, 'gatherer - last task not trace', k, '(%s)' % (
tasks[response.state.task.command.action])
need_to_trace = True
elif statuses[response.state.task.status.state] in [
'DELAYED', 'QUEUED', 'RUNNING'
]:
pnow = datetime.datetime.now()
print pnow, 'gatherer - trace still underway', k, '(%s)' % (
statuses[response.state.task.status.state])
need_to_trace = False
elif statuses[response.state.task.status.state] == 'FAILED':
pnow = datetime.datetime.now()
print pnow, 'gatherer - trace failed', k, '(%s)' % (
statuses[response.state.task.status.state])
need_to_trace = True
elif statuses[response.state.task.status.state] == 'SUCCESS':
# go and get the gzip'd trace dump
try:
response = bc_query.get_file(breadcrumbs, k,
response.state.task.status.id)
except:
break
statuses = {
v: k
for k, v in Common_pb2.TaskOutputResponse.Status.items()
}
if statuses[response.taskOutputResponse.status] != 'SUCCESS':
pnow = datetime.datetime.now()
print pnow, 'download failed'
continue
# decompress the trace dump
zfile = StringIO.StringIO()
zfile.write(response.taskOutputResponse.data)
zfile.seek(0)
try:
data = gzip.GzipFile(fileobj=zfile,
mode='rb').read().strip()
except:
traceback.print_exc()
pnow = datetime.datetime.now()
print pnow, 'gunzip failed'
continue
# parse the trace dump
lines = data.split('\n')[2:]
if len(lines) > 0:
if 'peer=' in lines[0]:
stat['nexthop_via'] = 'mesh' # mesh/failed backhaul
stat['nexthop_mac'] = lines[0].split('peer=')[1].split(
'/')[0].lower()
else:
stat['nexthop_via'] = 'wire' # ingress/has backhaul
stat['nexthop_mac'] = lines[0].split()[2].lower()
need_to_trace = True
if need_to_trace:
# request a trace task (only returns status of request)
try:
response = bc_query.get_trace(breadcrumbs, k, trace_ip)
time.sleep(5)
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - get_trace failed', k
break
if 'stats' not in breadcrumbs[k]:
breadcrumbs[k]['stats'] = []
breadcrumbs[k]['stats'] += [stat]
breadcrumbs[k]['stats'] = breadcrumbs[k]['stats'][-2:]
pnow = datetime.datetime.now()
print pnow, 'gatherer - got stats for', k
snooze = 8 - (datetime.datetime.now() - now).total_seconds()
time.sleep(snooze if snooze >= 0 else 0)
try:
bc_common.disconnect(breadcrumbs, k)
except:
pnow = datetime.datetime.now()
print pnow, 'gatherer - connect failed', k
time.sleep(5)
continue
time.sleep(4)
