def page_work(state_container, page_flip: bool):
'''Main page workhorse'''
if not state_container.searchSessionState:
state_container.searchSessionState = SearchSessionState()
state = state_container.searchSessionState
search_sidebar(state)
if page_flip or (state_container.search_text != state.search_text):
query_str = build_query(state, state_container.search_text)
else:
query_str = ''
if query_str:
if state.table == "address":
df = gui_get_df(state_container.sqobjs[state.table],
view="latest",
columns=['default'],
address=query_str.split())
else:
df = gui_get_df(state_container.sqobjs[state.table],
view="latest",
columns=['default'])
if not df.empty:
df = df.query(query_str).reset_index(drop=True)
expander = st.beta_expander(
f'Search for {state_container.search_text}', expanded=True)
with expander:
if not df.empty:
st.dataframe(df)
else:
st.info('No matching result found')
for count, prev_res in enumerate(reversed(state.prev_results)):
psrch, prev_df = prev_res
if psrch == state_container.search_text:
continue
expander = st.beta_expander(f'Search for {psrch}', expanded=True)
with expander:
if not prev_df.empty:
st.dataframe(prev_df)
else:
st.info('No matching result found')
if query_str and (state_container.search_text != state.search_text):
state.prev_results.append((state_container.search_text, df))
state.search_text = state_container.search_text
st.experimental_set_query_params(**asdict(state))
def draw_sidebar_status(state, sqobjs):
'''Draw appropriate sidebar for the page'''
do_refresh = st.sidebar.button('Refresh')
devdf = gui_get_df(sqobjs['device'], columns=['namespace', 'hostname'])
if devdf.empty:
st.error('Unable to retrieve any namespace info')
st.stop()
namespaces = [''] + sorted(devdf.namespace.unique().tolist())
if state.namespace:
nsidx = namespaces.index(state.namespace)
else:
nsidx = 0
namespace = st.sidebar.selectbox('Namespace', namespaces, index=nsidx)
if do_refresh:
st.caching.clear_cache()
st.sidebar.markdown(
'''This page provides an overview of the overall network status
Select one of the following pages from the Page menu to investigate further.
* __Xplore__: Look at all the data, look at summaries, run asserts, queries and more
* __Path__: Trace the paths between destinations in a given namespace
* __Search__: Search for addresses in various tables. Just type in any address you want to search. You can specify multiple addresses, space separated. See the search page for more help.
__Caching is enabled by default for 90 secs on all pages__. You can clear the cache by hitting the refresh button on this page or selecting "Clear Cache" option from the drop down menu on the top right hand corner
''')
if namespace != state.namespace:
state.namespace = namespace
def search_sidebar(state, sqobjs):
'''Draw the sidebar'''
devdf = gui_get_df(sqobjs['device'], columns=['namespace', 'hostname'])
if devdf.empty:
st.error('Unable to retrieve any namespace info')
st.stop()
namespaces = [''] + sorted(devdf.namespace.unique().tolist())
if state.nsidx == -1:
nsidx = 0
else:
nsidx = state.nsidx
namespace = st.sidebar.selectbox('Namespace', namespaces, index=nsidx)
st.sidebar.markdown("""Displays last 5 search results.
The search string can start with one of the following keywords: __address, route, mac, arpnd__, to specify which table you want the search to be performed in . If you don't specify a table name, address is assumed. For example, ```arpnd 172.16.1.101``` searches for entries with 172.16.1.101 in the IP address column of the arpnd table. Similarly, ```10.0.0.21``` searches for that IP address in the address table.
__In this initial release, you can only search for an IP address or MAC address__ in one of those tables. You can specify multiple addresses to look for by providing the addresses as a space separated values such as ```172.16.1.101 10.0.0.11``` or ```mac 00:01:02:03:04:05 00:21:22:23:24:25``` and so on. A combination of mac and IP address can also be specified. Obviously the combination doesn't apply to tables such as routes and macs. Support for more sophisticated search will be added in the next few releases.
""")
nsidx = namespaces.index(namespace)
if nsidx != state.nsidx:
state.nsidx = nsidx
return namespace
def path_sidebar(state, sqobjs):
"""Configure sidebar"""
devdf = gui_get_df(sqobjs['device'], columns=['namespace'])
if devdf.empty:
st.error('Unable to retrieve any namespace info')
st.stop()
namespaces = sorted(devdf.namespace.unique().tolist())
if state.namespace:
nsidx = namespaces.index(state.namespace)
else:
nsidx = 0
url = '&'.join([
f'{get_base_url()}?page=_Help_',
'help=yes',
'help_on=Path',
])
display_help_icon(url)
ok_button = st.sidebar.button('Trace')
namespace = st.sidebar.selectbox('Namespace',
namespaces, index=nsidx)
src_ph = st.sidebar.empty()
dst_ph = st.sidebar.empty()
state.source = src_ph.text_input('Source IP',
value=state.source)
state.dest = dst_ph.text_input('Dest IP', value=state.dest,
key='dest')
swap_src_dest = st.sidebar.button('Source <-> Dest')
if swap_src_dest:
source = src_ph.text_input('Source IP',
value=state.dest)
dest = dst_ph.text_input('Dest IP', value=state.source)
state.source = source
state.dest = dest
state.vrf = st.sidebar.text_input('VRF', value=state.vrf,
key='vrf')
state.start_time = st.sidebar.text_input('Start Time',
value=state.start_time,
key='start-time')
state.end_time = st.sidebar.text_input('End Time',
value=state.end_time,
key='end-time')
state.show_ifnames = st.sidebar.checkbox('Show in/out interface names',
value=state.show_ifnames)
if all(not x for x in [state.namespace,
state.source,
state.dest]):
state.run = False
elif ok_button:
state.run = True
elif namespace != state.namespace:
state.run = False
state.namespace = namespace
return
def get_failed_data(namespace: str, pgbar, sqobjs) -> FailedDFs:
'''Get interface/mlag/routing protocol states that are failed'''
faileddfs = FailedDFs()
progress = 40
for i, entry in enumerate(faileddfs.dfs):
entry['df'] = gui_get_df(sqobjs[entry['name']],
namespace=[namespace])
if not entry['df'].empty and (entry.get('query', '')):
entry['df'] = entry['df'].query(entry['query'])
pgbar.progress(progress + i*10)
return faileddfs
def page_work(state_container, page_flip: bool):
'''The main workhorse routine for the XNA page'''
if not state_container.statusSessionState:
state_container.statusSessionState = StatusSessionState()
state = state_container.statusSessionState
draw_sidebar_status(state, state_container.sqobjs)
col1, mid, col2 = st.beta_columns([2, 1, 2])
with col1:
dev_gr = st.empty()
with col2:
if_gr = st.empty()
col3, mid, col4 = st.beta_columns([2, 1, 2])
with col3:
bgp_gr = st.empty()
with col4:
ospf_gr = st.empty()
# Get each of the summarize info
if state.namespace:
ns = [state.namespace]
else:
ns = []
dev_df = gui_get_df(state_container.sqobjs['device'],
namespace=ns,
columns=['*'])
if not dev_df.empty:
dev_status = dev_df.groupby(by=['namespace', 'status'])['hostname'] \
.count() \
.reset_index() \
.rename({'hostname': 'count'}, axis=1)
dev_chart = alt.Chart(dev_status, title='Devices') \
.mark_bar(tooltip=True) \
.encode(y='status', x='count:Q', row='namespace',
color=alt.Color(
'status',
scale=alt.Scale(domain=['alive', 'dead'],
range=['green', 'red']))
)
dev_gr.altair_chart(dev_chart)
else:
dev_gr.info('No device info found')
if_df = gui_get_df(state_container.sqobjs['interfaces'],
namespace=ns,
columns=['*'])
if not if_df.empty:
if_df['state'] = np.where(
(if_df.state == "down") & (if_df.adminState == "down"),
"adminDown", if_df.state)
if_status = if_df.groupby(by=['namespace', 'state'])['hostname'] \
.count() \
.reset_index() \
.rename({'hostname': 'count'}, axis=1)
if_chart = alt.Chart(if_status, title='Interfaces') \
.mark_bar(tooltip=True) \
.encode(y='state', x='count:Q', row='namespace',
color=alt.Color(
'state',
scale=alt.Scale(domain=['up', 'adminDown',
'down'],
range=['green', 'orange',
'red']))
)
if_gr.altair_chart(if_chart)
else:
if_gr.info('No Interface info found')
bgp_df = gui_get_df(state_container.sqobjs['bgp'],
namespace=ns,
columns=['*'])
if not bgp_df.empty:
bgp_status = bgp_df.groupby(by=['namespace', 'state'])['hostname'] \
.count() \
.reset_index() \
.rename({'hostname': 'count'}, axis=1)
bgp_chart = alt.Chart(bgp_status, title='BGP') \
.mark_bar(tooltip=True) \
.encode(y='state', x='count:Q', row='namespace',
color=alt.Color(
'state',
scale=alt.Scale(
domain=['Established',
'NotEstd', 'dynamic'],
range=['green', 'red', 'orange']))
)
bgp_gr.altair_chart(bgp_chart)
ospf_df = gui_get_df(state_container.sqobjs['ospf'],
namespace=ns,
columns=['*'])
if not ospf_df.empty:
ospf_df['state'] = np.where(ospf_df.ifState == "adminDown",
"adminDown", ospf_df.adjState)
ospf_status = ospf_df.groupby(by=['namespace', 'state'])['hostname'] \
.count() \
.reset_index() \
.rename({'hostname': 'count'}, axis=1)
ospf_chart = alt.Chart(ospf_status, title='OSPF') \
.mark_bar(tooltip=True) \
.encode(y='state', x='count:Q', row='namespace',
color=alt.Color(
'state',
scale=alt.Scale(
domain=['full', 'fail',
'adminDown', 'passive'],
range=['green', 'red', 'orange', 'peach']))
)
ospf_gr.altair_chart(ospf_chart)
sqdf = gui_get_df(state_container.sqobjs['sqPoller'],
columns=['namespace', 'hostname', 'timestamp'],
service='device',
namespace=ns)
if not sqdf.empty:
hosts = sqdf.groupby(by=['namespace'])['hostname'] \
.nunique() \
.reset_index() \
.rename({'hostname': '#devices'}, axis=1)
times = sqdf.groupby(by=['namespace'])['timestamp'] \
.max().reset_index() \
.rename({'timestamp': 'lastPolledTime'}, axis=1)
pstats = times.merge(hosts, on=['namespace'])
st.subheader('Poller Status')
st.dataframe(pstats)
st.experimental_set_query_params(**{})
def page_work(state_container, page_flip: bool):
'''Main page workhorse'''
if not state_container.searchSessionState:
state_container.searchSessionState = SearchSessionState()
state = state_container.searchSessionState
namespace = search_sidebar(state, state_container.sqobjs)
query_str, uniq_dict = build_query(state, state_container.search_text)
if namespace:
query_ns = [namespace]
else:
query_ns = []
if query_str:
if state.table == "address":
df = gui_get_df(state_container.sqobjs[state.table],
namespace=query_ns,
view="latest",
columns=['default'],
address=query_str.split())
else:
df = gui_get_df(state_container.sqobjs[state.table],
namespace=query_ns,
view="latest",
columns=['default'])
if not df.empty:
df = df.query(query_str).reset_index(drop=True)
expander = st.beta_expander(
f'Search for {state_container.search_text}', expanded=True)
with expander:
if not df.empty:
st.dataframe(df)
else:
st.info('No matching result found')
elif uniq_dict:
columns = ['namespace'] + uniq_dict['column']
df = gui_get_df(state_container.sqobjs[uniq_dict['table']],
namespace=query_ns,
view='latest',
columns=columns)
if not df.empty:
df = df.groupby(by=columns).first().reset_index()
expander = st.beta_expander(
f'Search for {state_container.search_text}', expanded=True)
with expander:
if not df.empty:
st.dataframe(df)
else:
st.info('No matching result found')
for count, prev_res in enumerate(reversed(state.prev_results)):
psrch, prev_df = prev_res
if psrch == state_container.search_text:
continue
expander = st.beta_expander(f'Search for {psrch}', expanded=True)
with expander:
if not prev_df.empty:
st.dataframe(prev_df)
else:
st.info('No matching result found')
if ((query_str or uniq_dict)
and (state_container.search_text != state.search_text)):
state.prev_results.append((state_container.search_text, df))
state.search_text = state_container.search_text
st.experimental_set_query_params(**asdict(state))
def xplore_sidebar(state, sqobjs: dict):
'''Draw appropriate sidebar for the page'''
stime = state.start_time
etime = state.end_time
table_vals = sorted(list(sqobjs.keys()))
if state.table:
if isinstance(state.table, list):
tblidx = table_vals.index(state.table[0])
else:
tblidx = table_vals.index(state.table)
else:
tblidx = table_vals.index('device') # Default starting table
assert_val = state.assert_clicked
view_idx = 1 if state.view == 'all' else 0
devdf = gui_get_df(sqobjs['device'], columns=['namespace', 'hostname'])
if devdf.empty:
st.error('Unable to retrieve any namespace info')
st.stop()
namespaces = [""]
namespaces.extend(sorted(devdf.namespace.unique().tolist()))
if state.namespace:
nsidx = namespaces.index(state.namespace)
else:
nsidx = 0
namespace = st.sidebar.selectbox('Namespace', namespaces, index=nsidx)
if namespace != state.namespace:
state.hostname = None
state.namespace = namespace
hostnames = [""]
if state.namespace:
hostlist = devdf.query(f'namespace=="{state.namespace}"') \
.hostname.unique().tolist()
else:
hostlist = devdf.hostname.unique().tolist()
hostnames.extend(sorted(hostlist))
if state.hostname:
hostidx = hostnames.index(state.hostname)
else:
hostidx = 0
state.hostname = st.sidebar.selectbox('Hostname', hostnames, index=hostidx)
state.start_time = st.sidebar.text_input('Start time',
value=stime,
key='stime')
state.end_time = st.sidebar.text_input('End time',
value=etime,
key='etime')
table = st.sidebar.selectbox('Select Table to View',
tuple(table_vals),
index=tblidx)
if table != state.table:
# We need to reset the specific variables
state.query = ''
state.assert_clicked = False
state.uniq_clicked = 0
state.table = table
state.columns = 'default'
view_vals = ('latest', 'all')
if state.start_time and state.end_time:
# We show everything thats happened when both times are specified
view_idx = 1
state.view = st.sidebar.radio("View of Data", view_vals, index=view_idx)
fields = TablesObj().describe(table=state.table)
if state.table != 'tables':
colist = sorted((filter(lambda x: x not in ['index', 'sqvers'],
fields.name.tolist())))
columns = st.sidebar.multiselect('Pick columns',
['default', 'all'] + colist,
default=state.columns)
if ('default' in columns or 'all' in columns) and len(columns) == 1:
col_sel_val = True
else:
col_sel_val = False
col_ok = st.sidebar.checkbox('Column Selection Done',
value=col_sel_val)
if not col_ok:
columns = ['default']
else:
col_ok = True
columns = ['default']
if not columns:
columns = ['default']
state.columns = columns
if state.table in ['interfaces', 'ospf', 'bgp', 'evpnVni']:
state.assert_clicked = st.sidebar.checkbox('Run Assert',
value=assert_val)
else:
state.assert_clicked = False
if not col_ok:
st.experimental_set_query_params(**asdict(state))
st.stop()
if ('default' in columns or 'all' in columns) and len(columns) != 1:
st.error('Cannot select default/all with any other columns')
st.experimental_set_query_params(**asdict(state))
st.stop()
elif not columns:
st.error('Columns cannot be empty')
st.experimental_set_query_params(**asdict(state))
st.stop()
state.query = st.sidebar.text_input('Filter results with pandas query',
value=state.query,
key=state.table)
st.sidebar.markdown(
"[query syntax help](https://suzieq.readthedocs.io/en/latest/pandas-query-examples/)"
)
if columns == ['all']:
columns = ['*']
if state.table != "tables":
col_expander = st.sidebar.beta_expander('Column Names', expanded=False)
with col_expander:
st.subheader(f'{state.table} column names')
st.table(TablesObj().describe(
table=state.table).query('name != "sqvers"').reset_index(
drop=True).style)
def page_work(state_container, page_flip: bool):
'''The main workhorse routine for the Xplore page'''
if not state_container.xploreSessionState:
state_container.xploreSessionState = XploreSessionState()
state = state_container.xploreSessionState
state.columns = ['default']
else:
state = state_container.xploreSessionState
url_params = st.experimental_get_query_params()
page = url_params.pop('page', '')
if get_title() in page:
if url_params and not all(not x for x in url_params.values()):
for key in url_params:
if key == 'columns':
# This needs to be a list
continue
val = url_params.get(key, '')
if isinstance(val, list):
val = val[0]
url_params[key] = val
if key == '':
if val == 'True':
url_params[key] = True
else:
url_params[key] = False
state.__init__(**url_params)
sqobjs = state_container.sqobjs
# All the user input is preserved in the state vars
xplore_sidebar(state, sqobjs)
if state.table != "tables":
df = gui_get_df(sqobjs[state.table],
_table=state.table,
namespace=state.namespace.split(),
hostname=state.hostname.split(),
start_time=state.start_time,
end_time=state.end_time,
view=state.view,
columns=state.columns)
if state.table == "device" and 'uptime' in df.columns:
df.drop(columns=['uptime'], inplace=True)
else:
df = gui_get_df(sqobjs[state.table],
_table=state.table,
namespace=state.namespace.split(),
hostname=state.hostname.split(),
start_time=state.start_time,
end_time=state.end_time,
view=state.view)
query_str = ''
if not df.empty:
if 'error' in df.columns:
st.error(df.iloc[0].error)
st.experimental_set_query_params(**asdict(state))
st.stop()
if state.query:
try:
show_df = df.query(state.query)
query_str = state.query
except Exception:
st.warning('Query string throws an exception, ignoring')
show_df = df
query_str = ''
else:
show_df = df
else:
show_df = df
if state.table != "tables":
summ_df = xplore_run_summarize(sqobjs[state.table],
namespace=state.namespace.split(),
hostname=state.hostname.split(),
start_time=state.start_time,
end_time=state.end_time,
query_str=query_str)
else:
summ_df = pd.DataFrame()
if not show_df.empty:
dfcols = show_df.columns.tolist()
if (state.table == 'routes' and 'prefix' in dfcols
and 'prefixlen' not in dfcols):
dfcols.append('prefixlen')
dfcols = sorted((filter(lambda x: x not in ['index', 'sqvers'],
dfcols)))
grid1 = st.beta_container()
headercol, uniq_col = st.beta_columns(2)
with grid1:
with headercol:
st.write(
f'<h2 style="color: darkblue; font-weight: bold;">{state.table} View</h2>',
unsafe_allow_html=True)
if show_df.shape[0] > 256:
st.write(
f'Showing first 256 of {show_df.shape[0]} rows, use query to filter'
)
with uniq_col:
if state.table != "tables":
if (not state.uniq_clicked
or state.uniq_clicked not in dfcols):
if 'hostname' in dfcols:
selindex = dfcols.index('hostname') + 1
else:
selindex = 1
elif state.uniq_clicked in dfcols:
selindex = dfcols.index(state.uniq_clicked) + 1
state.uniq_clicked = st.selectbox('Distribution Count of',
options=['-'] + dfcols,
index=selindex,
key='distcount')
scol1, scol2 = st.beta_columns(2)
if state.table != "tables" and state.uniq_clicked != '-':
uniq_df = xplore_run_unique(show_df, columns=state.uniq_clicked)
else:
uniq_df = pd.DataFrame()
if state.assert_clicked:
assert_df = xplore_run_assert(sqobjs[state.table],
start_time=state.start_time,
end_time=state.end_time,
namespace=state.namespace.split())
else:
assert_df = pd.DataFrame()
if not summ_df.empty:
with scol1:
st.subheader('Summary Information')
st.dataframe(data=summ_df)
if not uniq_df.empty:
with scol2:
if uniq_df.shape[0] > 32:
st.warning(
f'{state.uniq_clicked} has cardinality > 32. Displaying top 32'
)
chart = alt.Chart(
uniq_df.head(32),
title=f'{state.uniq_clicked} Distribution') \
.mark_bar(color='purple', tooltip=True) \
.encode(y=alt.Y(f'{state.uniq_clicked}:N',
sort='-x'),
x='count')
else:
chart = alt.Chart(
uniq_df, title=f'{state.uniq_clicked} Distribution') \
.mark_bar(color='purple', tooltip=True) \
.encode(y=alt.Y(f'{state.uniq_clicked}:N',
sort='-x'),
x='count')
st.altair_chart(chart)
if state.table in ['interfaces', 'ospf', 'bgp', 'evpnVni']:
if assert_df.empty:
expand_assert = False
else:
expand_assert = True
validate_expander = st.beta_expander('Assert',
expanded=expand_assert)
with validate_expander:
if not assert_df.empty:
st.dataframe(data=assert_df)
elif state.assert_clicked:
st.write('Assert passed')
else:
st.write('Assert not run')
expander = st.beta_expander('Table', expanded=True)
with expander:
if not show_df.empty:
convert_dict = {
x: 'str'
for x in df.select_dtypes('category').columns
}
st.dataframe(data=sq_gui_style(
show_df.head(256).astype(convert_dict), state.table),
height=600,
width=2500)
else:
st.warning('No Data from query')
st.experimental_set_query_params(**asdict(state))
