def test_sqcmds_regex_namespace(table, datadir): cfgfile = create_dummy_config_file(datadir=datadir) df = get_sqobject(table)(config_file=cfgfile).get( hostname=['~leaf.*', '~exit.*'], namespace=['~ospf.*']) assert not df.empty if table == 'tables': assert df[df.table == 'device']['namespaces'].tolist() == [2] return if table in ['mlag', 'evpnVni', 'devconfig', 'bgp']: # why devconfig is empty for ospf-single needs investigation assert set(df.namespace.unique()) == set(['ospf-ibgp']) else: assert set(df.namespace.unique()) == set(['ospf-ibgp', 'ospf-single']) if table in ['network']: # network show has no hostname return if table not in ['mlag']: assert set(df.hostname.unique()) == set( ['leaf01', 'leaf02', 'leaf03', 'leaf04', 'exit01', 'exit02']) else: assert set(df.hostname.unique()) == set( ['leaf01', 'leaf02', 'leaf03', 'leaf04'])
def get_svc(command): """based on the command, find the module and service that the command is in return the service """ command_name = command svc = get_sqobject(command_name) return svc
def _coalescer_basic_test(pq_dir, namespace, path_src, path_dest): """Basic coalescer test Copy the parquet dir from the directory provided to a temp dir, coalesce and ensure that everything looks the same. This second part is done by ensuring table show looks the same before and after coalescing, and since we're assuming a run-once parquet input, there shouldn't be any duplicate entries that are being coalesced. We also run path before and after coalescing. path encompasseds many different tables and so with a single command we test multiple tables are correctly rendered. :param pq_dir: The original parquet dir :param namespace: The namespace to be used for checking info :param path_src: The source IP of the path :param path_dest: The destination IP of the path :returns: :rtype: """ temp_dir, tmpfile = _coalescer_init(pq_dir) tablesobj = get_sqobject('tables')(config_file=tmpfile.name) pre_tables_df = tablesobj.get() pathobj = get_sqobject('path')(config_file=tmpfile.name) pre_path_df = pathobj.get(namespace=[namespace], source=path_src, dest=path_dest) cfg = load_sq_config(config_file=tmpfile.name) do_coalesce(cfg, None) _verify_coalescing(temp_dir) post_tables_df = tablesobj.get() assert_df_equal(pre_tables_df, post_tables_df, None) post_path_df = pathobj.get(namespace=[namespace], source=path_src, dest=path_dest) assert_df_equal(pre_path_df, post_path_df, None) _coalescer_cleanup(temp_dir, tmpfile)
def test_schema_data_consistency(table, datadir, columns, get_table_data_cols): '''Test that all fields in dataframe and schema are consistent Only applies to show command for now It tests show columns=* is consistent with all fields in schema and that columns='default' matches all display fields in schema ''' if table in [ 'path', 'tables', 'ospfIf', 'ospfNbr', 'topcpu', 'topmem', 'ifCounters', 'time' ]: return df = get_table_data_cols # We have to get rid of false assertions. A bunch of data sets don't # have valid values for one or more tables. skip_table_data = { 'bgp': ['mixed'], 'ospf': ['vmx', 'basic_dual_bgp'], 'evpnVni': ['vmx', 'mixed', 'basic_dual_bgp'], 'mlag': ['vmx', 'junos', 'mixed'], 'devconfig': ['basic_dual_bgp'], } if df.empty: if table in skip_table_data: for x in skip_table_data[table]: if x in datadir: return elif table == "inventory" and 'vmx' not in datadir: return assert not df.empty sqobj = get_sqobject(table)() if columns == ['*']: schema_fld_set = set(sqobj.schema.fields) schema_fld_set.remove('sqvers') if table == 'bgp': schema_fld_set.remove('origPeer') elif table == "macs": schema_fld_set.remove('mackey') else: schema_fld_set = set(sqobj.schema.sorted_display_fields()) df_fld_set = set(df.columns) assert not schema_fld_set.symmetric_difference(df_fld_set)
def _render(self, layout: dict) -> None: state = self._state if not state.table: return sqobj = get_sqobject(state.table) try: df = gui_get_df(state.table, namespace=state.namespace.split(), start_time=state.start_time, end_time=state.end_time, view=state.view, columns=state.columns) except Exception as e: # pylint: disable=broad-except st.error(e) st.stop() if not df.empty: if 'error' in df.columns: st.error(df.iloc[0].error) st.experimental_set_query_params(**asdict(state)) st.stop() return else: st.info('No data returned by the table') st.experimental_set_query_params(**asdict(state)) st.stop() return if state.query: try: show_df = df.query(state.query).reset_index(drop=True) query_str = state.query except Exception as ex: # pylint: disable=broad-except st.error(f'Invalid query string: {ex}') st.stop() query_str = '' else: show_df = df query_str = '' if not show_df.empty: self._draw_summary_df(layout, sqobj, query_str) self._draw_assert_df(layout, sqobj) self._draw_table_df(layout, show_df)
def test_sqcmds_regex_hostname(table, datadir): cfgfile = create_dummy_config_file(datadir=datadir) df = get_sqobject(table)(config_file=cfgfile).get( hostname=['~leaf.*', '~exit.*']) if table == 'tables': if 'junos' in datadir: assert df[df.table == 'device']['deviceCnt'].tolist() == [4] elif not any(x in datadir for x in ['vmx', 'mixed']): # The hostnames for these output don't match the hostname regex assert df[df.table == 'device']['deviceCnt'].tolist() == [6] return if 'basic_dual_bgp' in datadir and table in [ 'ospf', 'evpnVni', 'devconfig' ]: return if not any(x in datadir for x in ['vmx', 'mixed', 'junos']): assert not df.empty if table in ['macs', 'vlan'] and 'basic_dual_bgp' in datadir: assert set(df.hostname.unique()) == set( ['leaf01', 'leaf02', 'leaf03', 'leaf04']) elif table not in ['mlag']: assert set(df.hostname.unique()) == set( ['leaf01', 'leaf02', 'leaf03', 'leaf04', 'exit01', 'exit02']) else: assert set(df.hostname.unique()) == set( ['leaf01', 'leaf02', 'leaf03', 'leaf04']) elif 'junos' in datadir: if table == 'mlag': # Our current Junos tests don't have MLAG return assert not df.empty if table == 'macs': assert set(df.hostname.unique()) == set(['leaf01', 'leaf02']) else: assert set(df.hostname.unique()) == set( ['leaf01', 'leaf02', 'exit01', 'exit02'])
def gui_get_df(table: str, verb: str = 'get', **kwargs) -> pd.DataFrame: """Get the cached value of the table provided The only verbs supported are get and find. Args: table ([str]): The table for which to get the data verb (str, optional): . Defaults to 'get'. Returns: [pandas.DataFrame]: The dataframe Raises: ValueError: If the verb is not supported """ view = kwargs.pop('view', 'latest') columns = kwargs.pop('columns', ['default']) stime = kwargs.pop('start_time', '') etime = kwargs.pop('end_time', '') sqobject = get_sqobject(table)(view=view, start_time=stime, end_time=etime) if columns == ['all']: columns = ['*'] if verb == 'get': df = sqobject.get(columns=columns, **kwargs) elif verb == 'find': df = sqobject.find(**kwargs) else: raise ValueError(f'Unsupported verb {verb}') if not df.empty: df = sqobject.humanize_fields(df) if table == 'address': if 'ipAddressList' in df.columns: df = df.explode('ipAddressList').fillna('') if 'ip6AddressList' in df.columns: df = df.explode('ip6AddressList').fillna('') if columns not in [['*'], ['default']]: return df[columns].reset_index(drop=True) return df.reset_index(drop=True)
def _get_table_sqobj(self, table: str, start_time: str = None, end_time: str = None, view=None): """Normalize pulling data from other tables into this one function Typically pulling data involves calling get_sqobject with a bunch of parameters that need to be passed to it, that a caller can forget to pass. A classic example is passing the view, start-time and end-time which is often forgotten. This function fixes this issue. Args: table (str): The table to retrieve the info from verb (str): The verb to use in the get_sqobject call """ return get_sqobject(table)(context=self.ctxt, start_time=start_time or self.iobj.start_time, end_time=end_time or self.iobj.end_time, view=view or self.iobj.view)
def __init__( self, engine: str = "", hostname: str = "", start_time: str = "", end_time: str = "", view: str = "", namespace: str = "", format: str = "", # pylint: disable=redefined-builtin columns: str = "default", query_str: str = ' ', ifname: str = '', state: str = '', type: str = '', vrf: str = '', mtu: str = '' ) -> None: super().__init__( engine=engine, hostname=hostname, start_time=start_time, end_time=end_time, view=view, namespace=namespace, columns=columns, format=format, query_str=query_str, sqobj=get_sqobject('interfaces') ) self.lvars = { 'ifname': ifname.split(), 'state': state, 'type': type.split(), 'vrf': vrf.split(), 'mtu': mtu.split(), }
def _render(self, layout: dict) -> None: '''Compute the path, render all objects''' state = self._state missing = [ x for x in [state.source, state.dest, state.namespace] if not x ] if missing: st.warning('Set namespace, source and destination in the sidebar ' 'to do a path trace ') st.stop() layout['pgbar'].progress(0) self._pathobj = get_sqobject('path')( config_file=st.session_state.config_file, start_time=state.start_time, end_time=state.end_time) try: df, summ_df = self._get_path(forward_dir=True) rdf = getattr(self._pathobj.engine, '_rdf', pd.DataFrame()) if not rdf.empty: # Something did get computed self._path_df = df except Exception as e: # pylint: disable=broad-except st.error(f'Invalid Input: {str(e)}') layout['pgbar'].progress(100) self._path_df = pd.DataFrame() st.stop() return layout['pgbar'].progress(40) if df.empty: layout['pgbar'].progress(100) st.info(f'No path to trace between {self._state.source} and ' f'{self._state.dest}') st.stop() return self._get_failed_data(state.namespace, layout['pgbar']) g = self._build_graphviz_obj(state.show_ifnames, df) layout['pgbar'].progress(100) # if not rev_df.empty: # rev_g = build_graphviz_obj(state, rev_df) layout['summary'].dataframe(data=summ_df.astype(str)) with layout['legend']: st.info('''Color Legend''') st.markdown(''' <b style="color:blue">Blue Lines</b> => L2 Hop(non-tunneled)<br> <b>Black Lines</b> => L3 Hop<br> <b style="color:purple">Purple lines</b> => Tunneled Hop<br> <b style="color:red">Red Lines</b> => Hops with Error<br> ''', unsafe_allow_html=True) layout['fw_path'].graphviz_chart(g, use_container_width=True) # rev_ph.graphviz_chart(rev_g, use_container_width=True) with layout['failed_tables']: for tbl, fdf in self._failed_dfs.items(): if not fdf.empty: table_expander = st.expander(f'Failed {tbl} Table', expanded=not fdf.empty) with table_expander: st.dataframe(fdf) with layout['table']: table_expander = st.expander('Path Table', expanded=True) with table_expander: self._draw_aggrid_df(df)
def test_transform(input_file): to_transform = Yaml2Class(input_file) try: data_directory = to_transform.transform.data_directory except AttributeError: print('Invalid transformation file, no data directory') pytest.fail('AttributeError', pytrace=True) # Make a copy of the data directory temp_dir, tmpfile = _coalescer_init(data_directory) cfg = load_sq_config(config_file=tmpfile.name) schemas = Schema(cfg['schema-directory']) for ele in to_transform.transform.transform: query_str_list = [] # Each transformation has a record => write's happen per record for record in ele.record: changed_fields = set() new_df = pd.DataFrame() tables = [x for x in dir(record) if not x.startswith('_')] for table in tables: # Lets read the data in now that we know the table tblobj = get_sqobject(table) pq_db = get_sqdb_engine(cfg, table, None, None) columns = schemas.fields_for_table(table) mod_df = tblobj(config_file=tmpfile.name).get(columns=columns) for key in getattr(record, table): query_str = key.match chg_df = pd.DataFrame() if query_str != "all": try: chg_df = mod_df.query(query_str) \ .reset_index(drop=True) except Exception as ex: assert (not ex) query_str_list.append(query_str) else: chg_df = mod_df _process_transform_set(key.set, chg_df, changed_fields) if new_df.empty: new_df = chg_df elif not chg_df.empty: new_df = pd.concat([new_df, chg_df]) if new_df.empty: continue # Write the records now _write_verify_transform(new_df, table, pq_db, SchemaForTable(table, schemas), tmpfile.name, query_str_list, changed_fields) # Now we coalesce and verify it works from suzieq.sqobjects.tables import TablesObj pre_table_df = TablesObj(config_file=tmpfile.name).get() do_coalesce(cfg, None) _verify_coalescing(temp_dir) post_table_df = TablesObj(config_file=tmpfile.name).get() assert_df_equal(pre_table_df, post_table_df, None) # Run additional tests on the coalesced data for ele in to_transform.transform.verify: table = [x for x in dir(ele) if not x.startswith('_')][0] tblobj = get_sqobject(table) for tst in getattr(ele, table): start_time = tst.test.get('start-time', '') end_time = tst.test.get('end-time', '') columns = tst.test.get('columns', ['default']) df = tblobj(config_file=tmpfile.name, start_time=start_time, end_time=end_time).get(columns=columns) if not df.empty and 'query' in tst.test: query_str = tst.test['query'] df = df.query(query_str).reset_index(drop=True) if 'assertempty' in tst.test: assert (df.empty) elif 'shape' in tst.test: shape = tst.test['shape'].split() if shape[0] != '*': assert (int(shape[0]) == df.shape[0]) if shape[1] != '*': assert (int(shape[1]) == df.shape[1]) else: assert (not df.empty) _coalescer_cleanup(temp_dir, tmpfile)
def _write_verify_transform(mod_df, table, dbeng, schema, config_file, query_str_list, changed_fields): """Write and verify that the written data is present :param mod_df: pd.DataFrame, the modified dataframe to write :param table: str, the name of the table to write :param dbeng: SqParquetDB, pointer to DB class to write/read :param schema: SchemaForTable, Schema of data to be written :param config_file: str, Filename where suzieq config is stored :param query_str_list: List[str], query string if any to apply to data for verification check :param changed_fields: set, list of changed fields to verify :returns: Nothing :rtype: """ mod_df = mod_df.reset_index(drop=True) mod_df.timestamp = mod_df.timestamp.astype(np.int64) mod_df.timestamp = mod_df.timestamp // 1000000 mod_df.sqvers = mod_df.sqvers.astype(str) dbeng.write(table, 'pandas', mod_df, False, schema.get_arrow_schema(), None) # Verify that what we wrote is what we got back mod_df.sqvers = mod_df.sqvers.astype(float) tblobj = get_sqobject(table) post_read_df = tblobj(config_file=config_file).get(columns=schema.fields) assert (not post_read_df.empty) # If the data was built up as a series of queries, we have to # apply the queries to verify that we have what we wrote dfconcat = None if query_str_list: for qstr in query_str_list: qdf = post_read_df.query(qstr).reset_index(drop=True) assert (not qdf.empty) if dfconcat is not None: dfconcat = pd.concat([dfconcat, qdf]) else: dfconcat = qdf if dfconcat is not None: qdf = dfconcat.set_index(schema.key_fields()) \ .sort_index() else: qdf = post_read_df.set_index(schema.key_fields()) \ .sort_index() mod_df = mod_df.set_index(schema.key_fields()) \ .query('~index.duplicated(keep="last")') \ .sort_index() mod_df.timestamp = humanize_timestamp(mod_df.timestamp, 'GMT') # We can't call assert_df_equal directly and so we # compare this way. The catch is if we accidentally # change some of the unchanged fields assert (mod_df.shape == qdf.shape) assert (not [ x for x in mod_df.columns.tolist() if x not in qdf.columns.tolist() ]) assert ((mod_df.index == qdf.index).all()) assert_df_equal(mod_df[changed_fields].reset_index(), qdf[changed_fields].reset_index(), None)
def test_rest_arg_consistency(service, verb): '''check that the arguments used in REST match whats in sqobjects''' alias_args = {'path': {'source': 'src'}} if verb == "describe" and not service == "tables": return if service in [ 'topcpu', 'topmem', 'ospfIf', 'ospfNbr', 'time', 'ifCounters' ]: return # import all relevant functions from the rest code first fnlist = list( filter(lambda x: x[0] == f'query_{service}_{verb}', inspect.getmembers(query, inspect.isfunction))) if not fnlist and service.endswith('s'): # Try the singular version fnlist = list( filter(lambda x: x[0] == f'query_{service[:-1]}_{verb}', inspect.getmembers(query, inspect.isfunction))) if fnlist: found_service_rest_fn = True else: found_service_rest_fn = False fnlist = list( filter(lambda x: x[0] == f'query_{service}', inspect.getmembers(query, inspect.isfunction))) if not fnlist and service.endswith('s'): # Try the singular version fnlist = list( filter(lambda x: x[0] == f'query_{service[:-1]}', inspect.getmembers(query, inspect.isfunction))) if not fnlist: assert fnlist, f"No functions found for {service}/{verb}" for fn in fnlist: rest_args = [] for i in inspect.getfullargspec(fn[1]).args: if i in ['verb', 'token', 'request']: continue aliases = alias_args.get(service, {}) val = i if i not in aliases else aliases[i] rest_args.append(val) sqobj = get_sqobject(service)() supported_verbs = { x[0].replace('aver', 'assert').replace('get', 'show') for x in inspect.getmembers(sqobj) if inspect.ismethod(x[1]) and not x[0].startswith('_') } if verb not in supported_verbs: continue aliases = alias_args.get(service, {}) arglist = getattr(sqobj, f'_valid_{verb}_args', None) if not arglist: if verb == "show": arglist = getattr(sqobj, '_valid_get_args', None) else: warnings.warn( f'Skipping arg check for {verb} in {service} due to ' f'missing valid_args list', category=ImportWarning) return arglist.extend([ 'namespace', 'hostname', 'start_time', 'end_time', 'format', 'view', 'columns', 'query_str' ]) valid_args = set(arglist) # In the tests below, we warn when we don't have the exact # {service}_{verb} REST function, which prevents us from picking the # correct set of args. for arg in valid_args: assert arg in rest_args, \ f"{arg} missing from {fn} arguments for verb {verb}" for arg in rest_args: if arg not in valid_args and arg != "status": # status is usually part of assert keyword and so ignore if found_service_rest_fn: assert False, \ f"{arg} not in {service} sqobj {verb} arguments" else: warnings.warn( f"{arg} not in {service} sqobj {verb} arguments", category=ImportWarning)
def _create_sidebar(self) -> None: state = self._state stime = state.start_time etime = state.end_time tables = filter( lambda x: x not in ['path', 'tables', 'ospfIf', 'ospfNbr', 'devconfig', 'topmem', 'topcpu', 'ifCounters', 'time'], get_tables() ) table_vals = [''] + sorted(tables) if state.table: if isinstance(state.table, list): tblidx = table_vals.index(state.table[0]) else: tblidx = table_vals.index(state.table) else: tblidx = table_vals.index('network') # Default starting table view_idx = 1 if state.view == 'all' else 0 devdf = gui_get_df('device', columns=['namespace', 'hostname']) if devdf.empty: st.error('Unable to retrieve any namespace info') st.stop() namespaces = [""] namespaces.extend(sorted(devdf.namespace.unique().tolist())) if state.namespace: nsidx = namespaces.index(state.namespace) else: nsidx = 0 with st.sidebar: with st.form('Xplore'): namespace = st.selectbox('Namespace', namespaces, key='xplore_namespace', index=nsidx) state.start_time = st.text_input('Start time', value=stime, key='xplore_stime') state.end_time = st.text_input('End time', value=etime, key='xplore_etime') table = st.selectbox( 'Select Table to View', tuple(table_vals), key='xplore_table', index=tblidx) if table != state.table: # We need to reset the specific variables state.query = '' state.assert_clicked = False state.uniq_clicked = '-' state.columns = ['default'] state.table = table view_vals = ('latest', 'all') if state.start_time and state.end_time: # Show everything thats happened if both times are given view_idx = 1 state.view = st.radio("View of Data", view_vals, index=view_idx, key='xplore_view') st.form_submit_button('Get', on_click=self._fetch_data) if namespace != state.namespace: state.namespace = namespace if state.table: tables_obj = get_sqobject('tables')(start_time=state.start_time, end_time=state.end_time, view=state.view) fields = tables_obj.describe(table=state.table) colist = sorted((filter(lambda x: x not in ['index', 'sqvers'], fields.name.tolist()))) columns = st.sidebar.multiselect('Pick columns', ['default', 'all'] + colist, key='xplore_columns', default=state.columns) col_sel_val = (('default' in columns or 'all' in columns) and len(columns) == 1) col_ok = st.sidebar.checkbox('Column Selection Done', key='xplore_col_done', value=col_sel_val) if not col_ok: columns = ['default'] else: col_ok = True columns = ['default'] if not columns: columns = ['default'] state.columns = columns state.experimental_ok = st.sidebar.checkbox( 'Enable Experimental Features', key='xplore_exp', on_change=self._sync_state) if state.table in ['interfaces', 'ospf', 'bgp', 'evpnVni']: state.assert_clicked = st.sidebar.checkbox( 'Run Assert', key='xplore_assert', on_change=self._sync_state) else: state.assert_clicked = False state.query = st.sidebar.text_input( 'Filter results with pandas query', value=state.query, key='xplore_query', on_change=self._sync_state) st.sidebar.markdown( "[query syntax help]" "(https://suzieq.readthedocs.io/en/latest/pandas-query-examples/)") if columns == ['all']: columns = ['*'] if state.table: col_expander = st.sidebar.expander('Column Names', expanded=False) with col_expander: st.subheader(f'{state.table} column names') st.table(tables_obj.describe(table=state.table) .query('name != "sqvers"') .reset_index(drop=True).astype(str).style) if not col_ok: st.experimental_set_query_params(**asdict(state)) st.stop() if ('default' in columns or 'all' in columns) and len(columns) != 1: st.error('Cannot select default/all with any other columns') st.experimental_set_query_params(**asdict(state)) st.stop() elif not columns: st.error('Columns cannot be empty') st.experimental_set_query_params(**asdict(state)) st.stop()