def test_sqcmds_regex_namespace(table, datadir):
cfgfile = create_dummy_config_file(datadir=datadir)
df = get_sqobject(table)(config_file=cfgfile).get(
hostname=['~leaf.*', '~exit.*'], namespace=['~ospf.*'])
assert not df.empty
if table == 'tables':
assert df[df.table == 'device']['namespaces'].tolist() == [2]
return
if table in ['mlag', 'evpnVni', 'devconfig', 'bgp']:
# why devconfig is empty for ospf-single needs investigation
assert set(df.namespace.unique()) == set(['ospf-ibgp'])
else:
assert set(df.namespace.unique()) == set(['ospf-ibgp', 'ospf-single'])
if table in ['network']:
# network show has no hostname
return
if table not in ['mlag']:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04', 'exit01', 'exit02'])
else:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04'])
def get_svc(command):
"""based on the command, find the module and service that the command is in
return the service
"""
command_name = command
svc = get_sqobject(command_name)
return svc
def _coalescer_basic_test(pq_dir, namespace, path_src, path_dest):
"""Basic coalescer test
Copy the parquet dir from the directory provided to a temp dir,
coalesce and ensure that everything looks the same. This second
part is done by ensuring table show looks the same before and
after coalescing, and since we're assuming a run-once parquet input,
there shouldn't be any duplicate entries that are being coalesced.
We also run path before and after coalescing. path encompasseds many
different tables and so with a single command we test multiple tables
are correctly rendered.
:param pq_dir: The original parquet dir
:param namespace: The namespace to be used for checking info
:param path_src: The source IP of the path
:param path_dest: The destination IP of the path
:returns:
:rtype:
"""
temp_dir, tmpfile = _coalescer_init(pq_dir)
tablesobj = get_sqobject('tables')(config_file=tmpfile.name)
pre_tables_df = tablesobj.get()
pathobj = get_sqobject('path')(config_file=tmpfile.name)
pre_path_df = pathobj.get(namespace=[namespace],
source=path_src,
dest=path_dest)
cfg = load_sq_config(config_file=tmpfile.name)
do_coalesce(cfg, None)
_verify_coalescing(temp_dir)
post_tables_df = tablesobj.get()
assert_df_equal(pre_tables_df, post_tables_df, None)
post_path_df = pathobj.get(namespace=[namespace],
source=path_src,
dest=path_dest)
assert_df_equal(pre_path_df, post_path_df, None)
_coalescer_cleanup(temp_dir, tmpfile)
def test_schema_data_consistency(table, datadir, columns, get_table_data_cols):
'''Test that all fields in dataframe and schema are consistent
Only applies to show command for now
It tests show columns=* is consistent with all fields in schema
and that columns='default' matches all display fields in schema
'''
if table in [
'path', 'tables', 'ospfIf', 'ospfNbr', 'topcpu', 'topmem',
'ifCounters', 'time'
]:
return
df = get_table_data_cols
# We have to get rid of false assertions. A bunch of data sets don't
# have valid values for one or more tables.
skip_table_data = {
'bgp': ['mixed'],
'ospf': ['vmx', 'basic_dual_bgp'],
'evpnVni': ['vmx', 'mixed', 'basic_dual_bgp'],
'mlag': ['vmx', 'junos', 'mixed'],
'devconfig': ['basic_dual_bgp'],
}
if df.empty:
if table in skip_table_data:
for x in skip_table_data[table]:
if x in datadir:
return
elif table == "inventory" and 'vmx' not in datadir:
return
assert not df.empty
sqobj = get_sqobject(table)()
if columns == ['*']:
schema_fld_set = set(sqobj.schema.fields)
schema_fld_set.remove('sqvers')
if table == 'bgp':
schema_fld_set.remove('origPeer')
elif table == "macs":
schema_fld_set.remove('mackey')
else:
schema_fld_set = set(sqobj.schema.sorted_display_fields())
df_fld_set = set(df.columns)
assert not schema_fld_set.symmetric_difference(df_fld_set)
def _render(self, layout: dict) -> None:
state = self._state
if not state.table:
return
sqobj = get_sqobject(state.table)
try:
df = gui_get_df(state.table,
namespace=state.namespace.split(),
start_time=state.start_time,
end_time=state.end_time,
view=state.view, columns=state.columns)
except Exception as e: # pylint: disable=broad-except
st.error(e)
st.stop()
if not df.empty:
if 'error' in df.columns:
st.error(df.iloc[0].error)
st.experimental_set_query_params(**asdict(state))
st.stop()
return
else:
st.info('No data returned by the table')
st.experimental_set_query_params(**asdict(state))
st.stop()
return
if state.query:
try:
show_df = df.query(state.query).reset_index(drop=True)
query_str = state.query
except Exception as ex: # pylint: disable=broad-except
st.error(f'Invalid query string: {ex}')
st.stop()
query_str = ''
else:
show_df = df
query_str = ''
if not show_df.empty:
self._draw_summary_df(layout, sqobj, query_str)
self._draw_assert_df(layout, sqobj)
self._draw_table_df(layout, show_df)
def test_sqcmds_regex_hostname(table, datadir):
cfgfile = create_dummy_config_file(datadir=datadir)
df = get_sqobject(table)(config_file=cfgfile).get(
hostname=['~leaf.*', '~exit.*'])
if table == 'tables':
if 'junos' in datadir:
assert df[df.table == 'device']['deviceCnt'].tolist() == [4]
elif not any(x in datadir for x in ['vmx', 'mixed']):
# The hostnames for these output don't match the hostname regex
assert df[df.table == 'device']['deviceCnt'].tolist() == [6]
return
if 'basic_dual_bgp' in datadir and table in [
'ospf', 'evpnVni', 'devconfig'
]:
return
if not any(x in datadir for x in ['vmx', 'mixed', 'junos']):
assert not df.empty
if table in ['macs', 'vlan'] and 'basic_dual_bgp' in datadir:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04'])
elif table not in ['mlag']:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04', 'exit01', 'exit02'])
else:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04'])
elif 'junos' in datadir:
if table == 'mlag':
# Our current Junos tests don't have MLAG
return
assert not df.empty
if table == 'macs':
assert set(df.hostname.unique()) == set(['leaf01', 'leaf02'])
else:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'exit01', 'exit02'])
def gui_get_df(table: str, verb: str = 'get', **kwargs) -> pd.DataFrame:
"""Get the cached value of the table provided
The only verbs supported are get and find.
Args:
table ([str]): The table for which to get the data
verb (str, optional): . Defaults to 'get'.
Returns:
[pandas.DataFrame]: The dataframe
Raises:
ValueError: If the verb is not supported
"""
view = kwargs.pop('view', 'latest')
columns = kwargs.pop('columns', ['default'])
stime = kwargs.pop('start_time', '')
etime = kwargs.pop('end_time', '')
sqobject = get_sqobject(table)(view=view, start_time=stime, end_time=etime)
if columns == ['all']:
columns = ['*']
if verb == 'get':
df = sqobject.get(columns=columns, **kwargs)
elif verb == 'find':
df = sqobject.find(**kwargs)
else:
raise ValueError(f'Unsupported verb {verb}')
if not df.empty:
df = sqobject.humanize_fields(df)
if table == 'address':
if 'ipAddressList' in df.columns:
df = df.explode('ipAddressList').fillna('')
if 'ip6AddressList' in df.columns:
df = df.explode('ip6AddressList').fillna('')
if columns not in [['*'], ['default']]:
return df[columns].reset_index(drop=True)
return df.reset_index(drop=True)
def _get_table_sqobj(self,
table: str,
start_time: str = None,
end_time: str = None,
view=None):
"""Normalize pulling data from other tables into this one function
Typically pulling data involves calling get_sqobject with a bunch of
parameters that need to be passed to it, that a caller can forget to
pass. A classic example is passing the view, start-time and end-time
which is often forgotten. This function fixes this issue.
Args:
table (str): The table to retrieve the info from
verb (str): The verb to use in the get_sqobject call
"""
return get_sqobject(table)(context=self.ctxt,
start_time=start_time
or self.iobj.start_time,
end_time=end_time or self.iobj.end_time,
view=view or self.iobj.view)
def __init__(
self,
engine: str = "",
hostname: str = "",
start_time: str = "",
end_time: str = "",
view: str = "",
namespace: str = "",
format: str = "", # pylint: disable=redefined-builtin
columns: str = "default",
query_str: str = ' ',
ifname: str = '',
state: str = '',
type: str = '',
vrf: str = '',
mtu: str = ''
) -> None:
super().__init__(
engine=engine,
hostname=hostname,
start_time=start_time,
end_time=end_time,
view=view,
namespace=namespace,
columns=columns,
format=format,
query_str=query_str,
sqobj=get_sqobject('interfaces')
)
self.lvars = {
'ifname': ifname.split(),
'state': state,
'type': type.split(),
'vrf': vrf.split(),
'mtu': mtu.split(),
}
def _render(self, layout: dict) -> None:
'''Compute the path, render all objects'''
state = self._state
missing = [
x for x in [state.source, state.dest, state.namespace] if not x
]
if missing:
st.warning('Set namespace, source and destination in the sidebar '
'to do a path trace ')
st.stop()
layout['pgbar'].progress(0)
self._pathobj = get_sqobject('path')(
config_file=st.session_state.config_file,
start_time=state.start_time,
end_time=state.end_time)
try:
df, summ_df = self._get_path(forward_dir=True)
rdf = getattr(self._pathobj.engine, '_rdf', pd.DataFrame())
if not rdf.empty:
# Something did get computed
self._path_df = df
except Exception as e: # pylint: disable=broad-except
st.error(f'Invalid Input: {str(e)}')
layout['pgbar'].progress(100)
self._path_df = pd.DataFrame()
st.stop()
return
layout['pgbar'].progress(40)
if df.empty:
layout['pgbar'].progress(100)
st.info(f'No path to trace between {self._state.source} and '
f'{self._state.dest}')
st.stop()
return
self._get_failed_data(state.namespace, layout['pgbar'])
g = self._build_graphviz_obj(state.show_ifnames, df)
layout['pgbar'].progress(100)
# if not rev_df.empty:
# rev_g = build_graphviz_obj(state, rev_df)
layout['summary'].dataframe(data=summ_df.astype(str))
with layout['legend']:
st.info('''Color Legend''')
st.markdown('''
<b style="color:blue">Blue Lines</b> => L2 Hop(non-tunneled)<br>
<b>Black Lines</b> => L3 Hop<br>
<b style="color:purple">Purple lines</b> => Tunneled Hop<br>
<b style="color:red">Red Lines</b> => Hops with Error<br>
''',
unsafe_allow_html=True)
layout['fw_path'].graphviz_chart(g, use_container_width=True)
# rev_ph.graphviz_chart(rev_g, use_container_width=True)
with layout['failed_tables']:
for tbl, fdf in self._failed_dfs.items():
if not fdf.empty:
table_expander = st.expander(f'Failed {tbl} Table',
expanded=not fdf.empty)
with table_expander:
st.dataframe(fdf)
with layout['table']:
table_expander = st.expander('Path Table', expanded=True)
with table_expander:
self._draw_aggrid_df(df)
def test_transform(input_file):
to_transform = Yaml2Class(input_file)
try:
data_directory = to_transform.transform.data_directory
except AttributeError:
print('Invalid transformation file, no data directory')
pytest.fail('AttributeError', pytrace=True)
# Make a copy of the data directory
temp_dir, tmpfile = _coalescer_init(data_directory)
cfg = load_sq_config(config_file=tmpfile.name)
schemas = Schema(cfg['schema-directory'])
for ele in to_transform.transform.transform:
query_str_list = []
# Each transformation has a record => write's happen per record
for record in ele.record:
changed_fields = set()
new_df = pd.DataFrame()
tables = [x for x in dir(record) if not x.startswith('_')]
for table in tables:
# Lets read the data in now that we know the table
tblobj = get_sqobject(table)
pq_db = get_sqdb_engine(cfg, table, None, None)
columns = schemas.fields_for_table(table)
mod_df = tblobj(config_file=tmpfile.name).get(columns=columns)
for key in getattr(record, table):
query_str = key.match
chg_df = pd.DataFrame()
if query_str != "all":
try:
chg_df = mod_df.query(query_str) \
.reset_index(drop=True)
except Exception as ex:
assert (not ex)
query_str_list.append(query_str)
else:
chg_df = mod_df
_process_transform_set(key.set, chg_df, changed_fields)
if new_df.empty:
new_df = chg_df
elif not chg_df.empty:
new_df = pd.concat([new_df, chg_df])
if new_df.empty:
continue
# Write the records now
_write_verify_transform(new_df, table, pq_db,
SchemaForTable(table,
schemas), tmpfile.name,
query_str_list, changed_fields)
# Now we coalesce and verify it works
from suzieq.sqobjects.tables import TablesObj
pre_table_df = TablesObj(config_file=tmpfile.name).get()
do_coalesce(cfg, None)
_verify_coalescing(temp_dir)
post_table_df = TablesObj(config_file=tmpfile.name).get()
assert_df_equal(pre_table_df, post_table_df, None)
# Run additional tests on the coalesced data
for ele in to_transform.transform.verify:
table = [x for x in dir(ele) if not x.startswith('_')][0]
tblobj = get_sqobject(table)
for tst in getattr(ele, table):
start_time = tst.test.get('start-time', '')
end_time = tst.test.get('end-time', '')
columns = tst.test.get('columns', ['default'])
df = tblobj(config_file=tmpfile.name,
start_time=start_time,
end_time=end_time).get(columns=columns)
if not df.empty and 'query' in tst.test:
query_str = tst.test['query']
df = df.query(query_str).reset_index(drop=True)
if 'assertempty' in tst.test:
assert (df.empty)
elif 'shape' in tst.test:
shape = tst.test['shape'].split()
if shape[0] != '*':
assert (int(shape[0]) == df.shape[0])
if shape[1] != '*':
assert (int(shape[1]) == df.shape[1])
else:
assert (not df.empty)
_coalescer_cleanup(temp_dir, tmpfile)
def _write_verify_transform(mod_df, table, dbeng, schema, config_file,
query_str_list, changed_fields):
"""Write and verify that the written data is present
:param mod_df: pd.DataFrame, the modified dataframe to write
:param table: str, the name of the table to write
:param dbeng: SqParquetDB, pointer to DB class to write/read
:param schema: SchemaForTable, Schema of data to be written
:param config_file: str, Filename where suzieq config is stored
:param query_str_list: List[str], query string if any to apply to data for
verification check
:param changed_fields: set, list of changed fields to verify
:returns: Nothing
:rtype:
"""
mod_df = mod_df.reset_index(drop=True)
mod_df.timestamp = mod_df.timestamp.astype(np.int64)
mod_df.timestamp = mod_df.timestamp // 1000000
mod_df.sqvers = mod_df.sqvers.astype(str)
dbeng.write(table, 'pandas', mod_df, False, schema.get_arrow_schema(),
None)
# Verify that what we wrote is what we got back
mod_df.sqvers = mod_df.sqvers.astype(float)
tblobj = get_sqobject(table)
post_read_df = tblobj(config_file=config_file).get(columns=schema.fields)
assert (not post_read_df.empty)
# If the data was built up as a series of queries, we have to
# apply the queries to verify that we have what we wrote
dfconcat = None
if query_str_list:
for qstr in query_str_list:
qdf = post_read_df.query(qstr).reset_index(drop=True)
assert (not qdf.empty)
if dfconcat is not None:
dfconcat = pd.concat([dfconcat, qdf])
else:
dfconcat = qdf
if dfconcat is not None:
qdf = dfconcat.set_index(schema.key_fields()) \
.sort_index()
else:
qdf = post_read_df.set_index(schema.key_fields()) \
.sort_index()
mod_df = mod_df.set_index(schema.key_fields()) \
.query('~index.duplicated(keep="last")') \
.sort_index()
mod_df.timestamp = humanize_timestamp(mod_df.timestamp, 'GMT')
# We can't call assert_df_equal directly and so we
# compare this way. The catch is if we accidentally
# change some of the unchanged fields
assert (mod_df.shape == qdf.shape)
assert (not [
x for x in mod_df.columns.tolist() if x not in qdf.columns.tolist()
])
assert ((mod_df.index == qdf.index).all())
assert_df_equal(mod_df[changed_fields].reset_index(),
qdf[changed_fields].reset_index(), None)
def test_rest_arg_consistency(service, verb):
'''check that the arguments used in REST match whats in sqobjects'''
alias_args = {'path': {'source': 'src'}}
if verb == "describe" and not service == "tables":
return
if service in [
'topcpu', 'topmem', 'ospfIf', 'ospfNbr', 'time', 'ifCounters'
]:
return
# import all relevant functions from the rest code first
fnlist = list(
filter(lambda x: x[0] == f'query_{service}_{verb}',
inspect.getmembers(query, inspect.isfunction)))
if not fnlist and service.endswith('s'):
# Try the singular version
fnlist = list(
filter(lambda x: x[0] == f'query_{service[:-1]}_{verb}',
inspect.getmembers(query, inspect.isfunction)))
if fnlist:
found_service_rest_fn = True
else:
found_service_rest_fn = False
fnlist = list(
filter(lambda x: x[0] == f'query_{service}',
inspect.getmembers(query, inspect.isfunction)))
if not fnlist and service.endswith('s'):
# Try the singular version
fnlist = list(
filter(lambda x: x[0] == f'query_{service[:-1]}',
inspect.getmembers(query, inspect.isfunction)))
if not fnlist:
assert fnlist, f"No functions found for {service}/{verb}"
for fn in fnlist:
rest_args = []
for i in inspect.getfullargspec(fn[1]).args:
if i in ['verb', 'token', 'request']:
continue
aliases = alias_args.get(service, {})
val = i if i not in aliases else aliases[i]
rest_args.append(val)
sqobj = get_sqobject(service)()
supported_verbs = {
x[0].replace('aver', 'assert').replace('get', 'show')
for x in inspect.getmembers(sqobj)
if inspect.ismethod(x[1]) and not x[0].startswith('_')
}
if verb not in supported_verbs:
continue
aliases = alias_args.get(service, {})
arglist = getattr(sqobj, f'_valid_{verb}_args', None)
if not arglist:
if verb == "show":
arglist = getattr(sqobj, '_valid_get_args', None)
else:
warnings.warn(
f'Skipping arg check for {verb} in {service} due to '
f'missing valid_args list',
category=ImportWarning)
return
arglist.extend([
'namespace', 'hostname', 'start_time', 'end_time', 'format',
'view', 'columns', 'query_str'
])
valid_args = set(arglist)
# In the tests below, we warn when we don't have the exact
# {service}_{verb} REST function, which prevents us from picking the
# correct set of args.
for arg in valid_args:
assert arg in rest_args, \
f"{arg} missing from {fn} arguments for verb {verb}"
for arg in rest_args:
if arg not in valid_args and arg != "status":
# status is usually part of assert keyword and so ignore
if found_service_rest_fn:
assert False, \
f"{arg} not in {service} sqobj {verb} arguments"
else:
warnings.warn(
f"{arg} not in {service} sqobj {verb} arguments",
category=ImportWarning)
def _create_sidebar(self) -> None:
state = self._state
stime = state.start_time
etime = state.end_time
tables = filter(
lambda x: x not in ['path', 'tables', 'ospfIf', 'ospfNbr',
'devconfig', 'topmem', 'topcpu', 'ifCounters',
'time'],
get_tables()
)
table_vals = [''] + sorted(tables)
if state.table:
if isinstance(state.table, list):
tblidx = table_vals.index(state.table[0])
else:
tblidx = table_vals.index(state.table)
else:
tblidx = table_vals.index('network') # Default starting table
view_idx = 1 if state.view == 'all' else 0
devdf = gui_get_df('device', columns=['namespace', 'hostname'])
if devdf.empty:
st.error('Unable to retrieve any namespace info')
st.stop()
namespaces = [""]
namespaces.extend(sorted(devdf.namespace.unique().tolist()))
if state.namespace:
nsidx = namespaces.index(state.namespace)
else:
nsidx = 0
with st.sidebar:
with st.form('Xplore'):
namespace = st.selectbox('Namespace',
namespaces, key='xplore_namespace',
index=nsidx)
state.start_time = st.text_input('Start time',
value=stime,
key='xplore_stime')
state.end_time = st.text_input('End time',
value=etime,
key='xplore_etime')
table = st.selectbox(
'Select Table to View', tuple(table_vals),
key='xplore_table', index=tblidx)
if table != state.table:
# We need to reset the specific variables
state.query = ''
state.assert_clicked = False
state.uniq_clicked = '-'
state.columns = ['default']
state.table = table
view_vals = ('latest', 'all')
if state.start_time and state.end_time:
# Show everything thats happened if both times are given
view_idx = 1
state.view = st.radio("View of Data", view_vals,
index=view_idx, key='xplore_view')
st.form_submit_button('Get', on_click=self._fetch_data)
if namespace != state.namespace:
state.namespace = namespace
if state.table:
tables_obj = get_sqobject('tables')(start_time=state.start_time,
end_time=state.end_time,
view=state.view)
fields = tables_obj.describe(table=state.table)
colist = sorted((filter(lambda x: x not in ['index', 'sqvers'],
fields.name.tolist())))
columns = st.sidebar.multiselect('Pick columns',
['default', 'all'] + colist,
key='xplore_columns',
default=state.columns)
col_sel_val = (('default' in columns or 'all' in columns)
and len(columns) == 1)
col_ok = st.sidebar.checkbox('Column Selection Done',
key='xplore_col_done',
value=col_sel_val)
if not col_ok:
columns = ['default']
else:
col_ok = True
columns = ['default']
if not columns:
columns = ['default']
state.columns = columns
state.experimental_ok = st.sidebar.checkbox(
'Enable Experimental Features',
key='xplore_exp',
on_change=self._sync_state)
if state.table in ['interfaces', 'ospf', 'bgp', 'evpnVni']:
state.assert_clicked = st.sidebar.checkbox(
'Run Assert', key='xplore_assert',
on_change=self._sync_state)
else:
state.assert_clicked = False
state.query = st.sidebar.text_input(
'Filter results with pandas query', value=state.query,
key='xplore_query', on_change=self._sync_state)
st.sidebar.markdown(
"[query syntax help]"
"(https://suzieq.readthedocs.io/en/latest/pandas-query-examples/)")
if columns == ['all']:
columns = ['*']
if state.table:
col_expander = st.sidebar.expander('Column Names', expanded=False)
with col_expander:
st.subheader(f'{state.table} column names')
st.table(tables_obj.describe(table=state.table)
.query('name != "sqvers"')
.reset_index(drop=True).astype(str).style)
if not col_ok:
st.experimental_set_query_params(**asdict(state))
st.stop()
if ('default' in columns or 'all' in columns) and len(columns) != 1:
st.error('Cannot select default/all with any other columns')
st.experimental_set_query_params(**asdict(state))
st.stop()
elif not columns:
st.error('Columns cannot be empty')
st.experimental_set_query_params(**asdict(state))
st.stop()
