def page_work(state_container, page_flip: bool): '''Main page workhorse''' if not state_container.searchSessionState: state_container.searchSessionState = SearchSessionState() state = state_container.searchSessionState search_sidebar(state) if page_flip or (state_container.search_text != state.search_text): query_str = build_query(state, state_container.search_text) else: query_str = '' if query_str: if state.table == "address": df = gui_get_df(state_container.sqobjs[state.table], view="latest", columns=['default'], address=query_str.split()) else: df = gui_get_df(state_container.sqobjs[state.table], view="latest", columns=['default']) if not df.empty: df = df.query(query_str).reset_index(drop=True) expander = st.beta_expander( f'Search for {state_container.search_text}', expanded=True) with expander: if not df.empty: st.dataframe(df) else: st.info('No matching result found') for count, prev_res in enumerate(reversed(state.prev_results)): psrch, prev_df = prev_res if psrch == state_container.search_text: continue expander = st.beta_expander(f'Search for {psrch}', expanded=True) with expander: if not prev_df.empty: st.dataframe(prev_df) else: st.info('No matching result found') if query_str and (state_container.search_text != state.search_text): state.prev_results.append((state_container.search_text, df)) state.search_text = state_container.search_text st.experimental_set_query_params(**asdict(state))
def draw_sidebar_status(state, sqobjs): '''Draw appropriate sidebar for the page''' do_refresh = st.sidebar.button('Refresh') devdf = gui_get_df(sqobjs['device'], columns=['namespace', 'hostname']) if devdf.empty: st.error('Unable to retrieve any namespace info') st.stop() namespaces = [''] + sorted(devdf.namespace.unique().tolist()) if state.namespace: nsidx = namespaces.index(state.namespace) else: nsidx = 0 namespace = st.sidebar.selectbox('Namespace', namespaces, index=nsidx) if do_refresh: st.caching.clear_cache() st.sidebar.markdown( '''This page provides an overview of the overall network status Select one of the following pages from the Page menu to investigate further. * __Xplore__: Look at all the data, look at summaries, run asserts, queries and more * __Path__: Trace the paths between destinations in a given namespace * __Search__: Search for addresses in various tables. Just type in any address you want to search. You can specify multiple addresses, space separated. See the search page for more help. __Caching is enabled by default for 90 secs on all pages__. You can clear the cache by hitting the refresh button on this page or selecting "Clear Cache" option from the drop down menu on the top right hand corner ''') if namespace != state.namespace: state.namespace = namespace
def search_sidebar(state, sqobjs): '''Draw the sidebar''' devdf = gui_get_df(sqobjs['device'], columns=['namespace', 'hostname']) if devdf.empty: st.error('Unable to retrieve any namespace info') st.stop() namespaces = [''] + sorted(devdf.namespace.unique().tolist()) if state.nsidx == -1: nsidx = 0 else: nsidx = state.nsidx namespace = st.sidebar.selectbox('Namespace', namespaces, index=nsidx) st.sidebar.markdown("""Displays last 5 search results. The search string can start with one of the following keywords: __address, route, mac, arpnd__, to specify which table you want the search to be performed in . If you don't specify a table name, address is assumed. For example, ```arpnd 172.16.1.101``` searches for entries with 172.16.1.101 in the IP address column of the arpnd table. Similarly, ```10.0.0.21``` searches for that IP address in the address table. __In this initial release, you can only search for an IP address or MAC address__ in one of those tables. You can specify multiple addresses to look for by providing the addresses as a space separated values such as ```172.16.1.101 10.0.0.11``` or ```mac 00:01:02:03:04:05 00:21:22:23:24:25``` and so on. A combination of mac and IP address can also be specified. Obviously the combination doesn't apply to tables such as routes and macs. Support for more sophisticated search will be added in the next few releases. """) nsidx = namespaces.index(namespace) if nsidx != state.nsidx: state.nsidx = nsidx return namespace
def path_sidebar(state, sqobjs): """Configure sidebar""" devdf = gui_get_df(sqobjs['device'], columns=['namespace']) if devdf.empty: st.error('Unable to retrieve any namespace info') st.stop() namespaces = sorted(devdf.namespace.unique().tolist()) if state.namespace: nsidx = namespaces.index(state.namespace) else: nsidx = 0 url = '&'.join([ f'{get_base_url()}?page=_Help_', 'help=yes', 'help_on=Path', ]) display_help_icon(url) ok_button = st.sidebar.button('Trace') namespace = st.sidebar.selectbox('Namespace', namespaces, index=nsidx) src_ph = st.sidebar.empty() dst_ph = st.sidebar.empty() state.source = src_ph.text_input('Source IP', value=state.source) state.dest = dst_ph.text_input('Dest IP', value=state.dest, key='dest') swap_src_dest = st.sidebar.button('Source <-> Dest') if swap_src_dest: source = src_ph.text_input('Source IP', value=state.dest) dest = dst_ph.text_input('Dest IP', value=state.source) state.source = source state.dest = dest state.vrf = st.sidebar.text_input('VRF', value=state.vrf, key='vrf') state.start_time = st.sidebar.text_input('Start Time', value=state.start_time, key='start-time') state.end_time = st.sidebar.text_input('End Time', value=state.end_time, key='end-time') state.show_ifnames = st.sidebar.checkbox('Show in/out interface names', value=state.show_ifnames) if all(not x for x in [state.namespace, state.source, state.dest]): state.run = False elif ok_button: state.run = True elif namespace != state.namespace: state.run = False state.namespace = namespace return
def get_failed_data(namespace: str, pgbar, sqobjs) -> FailedDFs: '''Get interface/mlag/routing protocol states that are failed''' faileddfs = FailedDFs() progress = 40 for i, entry in enumerate(faileddfs.dfs): entry['df'] = gui_get_df(sqobjs[entry['name']], namespace=[namespace]) if not entry['df'].empty and (entry.get('query', '')): entry['df'] = entry['df'].query(entry['query']) pgbar.progress(progress + i*10) return faileddfs
def page_work(state_container, page_flip: bool): '''The main workhorse routine for the XNA page''' if not state_container.statusSessionState: state_container.statusSessionState = StatusSessionState() state = state_container.statusSessionState draw_sidebar_status(state, state_container.sqobjs) col1, mid, col2 = st.beta_columns([2, 1, 2]) with col1: dev_gr = st.empty() with col2: if_gr = st.empty() col3, mid, col4 = st.beta_columns([2, 1, 2]) with col3: bgp_gr = st.empty() with col4: ospf_gr = st.empty() # Get each of the summarize info if state.namespace: ns = [state.namespace] else: ns = [] dev_df = gui_get_df(state_container.sqobjs['device'], namespace=ns, columns=['*']) if not dev_df.empty: dev_status = dev_df.groupby(by=['namespace', 'status'])['hostname'] \ .count() \ .reset_index() \ .rename({'hostname': 'count'}, axis=1) dev_chart = alt.Chart(dev_status, title='Devices') \ .mark_bar(tooltip=True) \ .encode(y='status', x='count:Q', row='namespace', color=alt.Color( 'status', scale=alt.Scale(domain=['alive', 'dead'], range=['green', 'red'])) ) dev_gr.altair_chart(dev_chart) else: dev_gr.info('No device info found') if_df = gui_get_df(state_container.sqobjs['interfaces'], namespace=ns, columns=['*']) if not if_df.empty: if_df['state'] = np.where( (if_df.state == "down") & (if_df.adminState == "down"), "adminDown", if_df.state) if_status = if_df.groupby(by=['namespace', 'state'])['hostname'] \ .count() \ .reset_index() \ .rename({'hostname': 'count'}, axis=1) if_chart = alt.Chart(if_status, title='Interfaces') \ .mark_bar(tooltip=True) \ .encode(y='state', x='count:Q', row='namespace', color=alt.Color( 'state', scale=alt.Scale(domain=['up', 'adminDown', 'down'], range=['green', 'orange', 'red'])) ) if_gr.altair_chart(if_chart) else: if_gr.info('No Interface info found') bgp_df = gui_get_df(state_container.sqobjs['bgp'], namespace=ns, columns=['*']) if not bgp_df.empty: bgp_status = bgp_df.groupby(by=['namespace', 'state'])['hostname'] \ .count() \ .reset_index() \ .rename({'hostname': 'count'}, axis=1) bgp_chart = alt.Chart(bgp_status, title='BGP') \ .mark_bar(tooltip=True) \ .encode(y='state', x='count:Q', row='namespace', color=alt.Color( 'state', scale=alt.Scale( domain=['Established', 'NotEstd', 'dynamic'], range=['green', 'red', 'orange'])) ) bgp_gr.altair_chart(bgp_chart) ospf_df = gui_get_df(state_container.sqobjs['ospf'], namespace=ns, columns=['*']) if not ospf_df.empty: ospf_df['state'] = np.where(ospf_df.ifState == "adminDown", "adminDown", ospf_df.adjState) ospf_status = ospf_df.groupby(by=['namespace', 'state'])['hostname'] \ .count() \ .reset_index() \ .rename({'hostname': 'count'}, axis=1) ospf_chart = alt.Chart(ospf_status, title='OSPF') \ .mark_bar(tooltip=True) \ .encode(y='state', x='count:Q', row='namespace', color=alt.Color( 'state', scale=alt.Scale( domain=['full', 'fail', 'adminDown', 'passive'], range=['green', 'red', 'orange', 'peach'])) ) ospf_gr.altair_chart(ospf_chart) sqdf = gui_get_df(state_container.sqobjs['sqPoller'], columns=['namespace', 'hostname', 'timestamp'], service='device', namespace=ns) if not sqdf.empty: hosts = sqdf.groupby(by=['namespace'])['hostname'] \ .nunique() \ .reset_index() \ .rename({'hostname': '#devices'}, axis=1) times = sqdf.groupby(by=['namespace'])['timestamp'] \ .max().reset_index() \ .rename({'timestamp': 'lastPolledTime'}, axis=1) pstats = times.merge(hosts, on=['namespace']) st.subheader('Poller Status') st.dataframe(pstats) st.experimental_set_query_params(**{})
def page_work(state_container, page_flip: bool): '''Main page workhorse''' if not state_container.searchSessionState: state_container.searchSessionState = SearchSessionState() state = state_container.searchSessionState namespace = search_sidebar(state, state_container.sqobjs) query_str, uniq_dict = build_query(state, state_container.search_text) if namespace: query_ns = [namespace] else: query_ns = [] if query_str: if state.table == "address": df = gui_get_df(state_container.sqobjs[state.table], namespace=query_ns, view="latest", columns=['default'], address=query_str.split()) else: df = gui_get_df(state_container.sqobjs[state.table], namespace=query_ns, view="latest", columns=['default']) if not df.empty: df = df.query(query_str).reset_index(drop=True) expander = st.beta_expander( f'Search for {state_container.search_text}', expanded=True) with expander: if not df.empty: st.dataframe(df) else: st.info('No matching result found') elif uniq_dict: columns = ['namespace'] + uniq_dict['column'] df = gui_get_df(state_container.sqobjs[uniq_dict['table']], namespace=query_ns, view='latest', columns=columns) if not df.empty: df = df.groupby(by=columns).first().reset_index() expander = st.beta_expander( f'Search for {state_container.search_text}', expanded=True) with expander: if not df.empty: st.dataframe(df) else: st.info('No matching result found') for count, prev_res in enumerate(reversed(state.prev_results)): psrch, prev_df = prev_res if psrch == state_container.search_text: continue expander = st.beta_expander(f'Search for {psrch}', expanded=True) with expander: if not prev_df.empty: st.dataframe(prev_df) else: st.info('No matching result found') if ((query_str or uniq_dict) and (state_container.search_text != state.search_text)): state.prev_results.append((state_container.search_text, df)) state.search_text = state_container.search_text st.experimental_set_query_params(**asdict(state))
def xplore_sidebar(state, sqobjs: dict): '''Draw appropriate sidebar for the page''' stime = state.start_time etime = state.end_time table_vals = sorted(list(sqobjs.keys())) if state.table: if isinstance(state.table, list): tblidx = table_vals.index(state.table[0]) else: tblidx = table_vals.index(state.table) else: tblidx = table_vals.index('device') # Default starting table assert_val = state.assert_clicked view_idx = 1 if state.view == 'all' else 0 devdf = gui_get_df(sqobjs['device'], columns=['namespace', 'hostname']) if devdf.empty: st.error('Unable to retrieve any namespace info') st.stop() namespaces = [""] namespaces.extend(sorted(devdf.namespace.unique().tolist())) if state.namespace: nsidx = namespaces.index(state.namespace) else: nsidx = 0 namespace = st.sidebar.selectbox('Namespace', namespaces, index=nsidx) if namespace != state.namespace: state.hostname = None state.namespace = namespace hostnames = [""] if state.namespace: hostlist = devdf.query(f'namespace=="{state.namespace}"') \ .hostname.unique().tolist() else: hostlist = devdf.hostname.unique().tolist() hostnames.extend(sorted(hostlist)) if state.hostname: hostidx = hostnames.index(state.hostname) else: hostidx = 0 state.hostname = st.sidebar.selectbox('Hostname', hostnames, index=hostidx) state.start_time = st.sidebar.text_input('Start time', value=stime, key='stime') state.end_time = st.sidebar.text_input('End time', value=etime, key='etime') table = st.sidebar.selectbox('Select Table to View', tuple(table_vals), index=tblidx) if table != state.table: # We need to reset the specific variables state.query = '' state.assert_clicked = False state.uniq_clicked = 0 state.table = table state.columns = 'default' view_vals = ('latest', 'all') if state.start_time and state.end_time: # We show everything thats happened when both times are specified view_idx = 1 state.view = st.sidebar.radio("View of Data", view_vals, index=view_idx) fields = TablesObj().describe(table=state.table) if state.table != 'tables': colist = sorted((filter(lambda x: x not in ['index', 'sqvers'], fields.name.tolist()))) columns = st.sidebar.multiselect('Pick columns', ['default', 'all'] + colist, default=state.columns) if ('default' in columns or 'all' in columns) and len(columns) == 1: col_sel_val = True else: col_sel_val = False col_ok = st.sidebar.checkbox('Column Selection Done', value=col_sel_val) if not col_ok: columns = ['default'] else: col_ok = True columns = ['default'] if not columns: columns = ['default'] state.columns = columns if state.table in ['interfaces', 'ospf', 'bgp', 'evpnVni']: state.assert_clicked = st.sidebar.checkbox('Run Assert', value=assert_val) else: state.assert_clicked = False if not col_ok: st.experimental_set_query_params(**asdict(state)) st.stop() if ('default' in columns or 'all' in columns) and len(columns) != 1: st.error('Cannot select default/all with any other columns') st.experimental_set_query_params(**asdict(state)) st.stop() elif not columns: st.error('Columns cannot be empty') st.experimental_set_query_params(**asdict(state)) st.stop() state.query = st.sidebar.text_input('Filter results with pandas query', value=state.query, key=state.table) st.sidebar.markdown( "[query syntax help](https://suzieq.readthedocs.io/en/latest/pandas-query-examples/)" ) if columns == ['all']: columns = ['*'] if state.table != "tables": col_expander = st.sidebar.beta_expander('Column Names', expanded=False) with col_expander: st.subheader(f'{state.table} column names') st.table(TablesObj().describe( table=state.table).query('name != "sqvers"').reset_index( drop=True).style)
def page_work(state_container, page_flip: bool): '''The main workhorse routine for the Xplore page''' if not state_container.xploreSessionState: state_container.xploreSessionState = XploreSessionState() state = state_container.xploreSessionState state.columns = ['default'] else: state = state_container.xploreSessionState url_params = st.experimental_get_query_params() page = url_params.pop('page', '') if get_title() in page: if url_params and not all(not x for x in url_params.values()): for key in url_params: if key == 'columns': # This needs to be a list continue val = url_params.get(key, '') if isinstance(val, list): val = val[0] url_params[key] = val if key == '': if val == 'True': url_params[key] = True else: url_params[key] = False state.__init__(**url_params) sqobjs = state_container.sqobjs # All the user input is preserved in the state vars xplore_sidebar(state, sqobjs) if state.table != "tables": df = gui_get_df(sqobjs[state.table], _table=state.table, namespace=state.namespace.split(), hostname=state.hostname.split(), start_time=state.start_time, end_time=state.end_time, view=state.view, columns=state.columns) if state.table == "device" and 'uptime' in df.columns: df.drop(columns=['uptime'], inplace=True) else: df = gui_get_df(sqobjs[state.table], _table=state.table, namespace=state.namespace.split(), hostname=state.hostname.split(), start_time=state.start_time, end_time=state.end_time, view=state.view) query_str = '' if not df.empty: if 'error' in df.columns: st.error(df.iloc[0].error) st.experimental_set_query_params(**asdict(state)) st.stop() if state.query: try: show_df = df.query(state.query) query_str = state.query except Exception: st.warning('Query string throws an exception, ignoring') show_df = df query_str = '' else: show_df = df else: show_df = df if state.table != "tables": summ_df = xplore_run_summarize(sqobjs[state.table], namespace=state.namespace.split(), hostname=state.hostname.split(), start_time=state.start_time, end_time=state.end_time, query_str=query_str) else: summ_df = pd.DataFrame() if not show_df.empty: dfcols = show_df.columns.tolist() if (state.table == 'routes' and 'prefix' in dfcols and 'prefixlen' not in dfcols): dfcols.append('prefixlen') dfcols = sorted((filter(lambda x: x not in ['index', 'sqvers'], dfcols))) grid1 = st.beta_container() headercol, uniq_col = st.beta_columns(2) with grid1: with headercol: st.write( f'<h2 style="color: darkblue; font-weight: bold;">{state.table} View</h2>', unsafe_allow_html=True) if show_df.shape[0] > 256: st.write( f'Showing first 256 of {show_df.shape[0]} rows, use query to filter' ) with uniq_col: if state.table != "tables": if (not state.uniq_clicked or state.uniq_clicked not in dfcols): if 'hostname' in dfcols: selindex = dfcols.index('hostname') + 1 else: selindex = 1 elif state.uniq_clicked in dfcols: selindex = dfcols.index(state.uniq_clicked) + 1 state.uniq_clicked = st.selectbox('Distribution Count of', options=['-'] + dfcols, index=selindex, key='distcount') scol1, scol2 = st.beta_columns(2) if state.table != "tables" and state.uniq_clicked != '-': uniq_df = xplore_run_unique(show_df, columns=state.uniq_clicked) else: uniq_df = pd.DataFrame() if state.assert_clicked: assert_df = xplore_run_assert(sqobjs[state.table], start_time=state.start_time, end_time=state.end_time, namespace=state.namespace.split()) else: assert_df = pd.DataFrame() if not summ_df.empty: with scol1: st.subheader('Summary Information') st.dataframe(data=summ_df) if not uniq_df.empty: with scol2: if uniq_df.shape[0] > 32: st.warning( f'{state.uniq_clicked} has cardinality > 32. Displaying top 32' ) chart = alt.Chart( uniq_df.head(32), title=f'{state.uniq_clicked} Distribution') \ .mark_bar(color='purple', tooltip=True) \ .encode(y=alt.Y(f'{state.uniq_clicked}:N', sort='-x'), x='count') else: chart = alt.Chart( uniq_df, title=f'{state.uniq_clicked} Distribution') \ .mark_bar(color='purple', tooltip=True) \ .encode(y=alt.Y(f'{state.uniq_clicked}:N', sort='-x'), x='count') st.altair_chart(chart) if state.table in ['interfaces', 'ospf', 'bgp', 'evpnVni']: if assert_df.empty: expand_assert = False else: expand_assert = True validate_expander = st.beta_expander('Assert', expanded=expand_assert) with validate_expander: if not assert_df.empty: st.dataframe(data=assert_df) elif state.assert_clicked: st.write('Assert passed') else: st.write('Assert not run') expander = st.beta_expander('Table', expanded=True) with expander: if not show_df.empty: convert_dict = { x: 'str' for x in df.select_dtypes('category').columns } st.dataframe(data=sq_gui_style( show_df.head(256).astype(convert_dict), state.table), height=600, width=2500) else: st.warning('No Data from query') st.experimental_set_query_params(**asdict(state))