def test_sqcmds_regex_namespace(table, datadir):
cfgfile = create_dummy_config_file(datadir=datadir)
df = get_sqobject(table)(config_file=cfgfile).get(
hostname=['~leaf.*', '~exit.*'], namespace=['~ospf.*'])
assert not df.empty
if table == 'tables':
assert df[df.table == 'device']['namespaces'].tolist() == [2]
return
if table in ['mlag', 'evpnVni', 'devconfig', 'bgp']:
# why devconfig is empty for ospf-single needs investigation
assert set(df.namespace.unique()) == set(['ospf-ibgp'])
else:
assert set(df.namespace.unique()) == set(['ospf-ibgp', 'ospf-single'])
if table in ['network']:
# network show has no hostname
return
if table not in ['mlag']:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04', 'exit01', 'exit02'])
else:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04'])
def _coalescer_init(pq_dir: str):
"""Basic Coalescer test
:param pq_dir: the input parquet dir to be copied, this is the root of the
parquet dir (tests/data/nxos/parquet-out, for example)
:returns: temporary dir where the parquet data has been copied to
:rtype: TemporaryDirectory
:returns: Temporary config file
:rtype: NamedTemporaryFile
"""
# Create a temp dir
# pylint: disable=consider-using-with
temp_dir = TemporaryDirectory()
# Copy the directory we want to copy
copy_tree(pq_dir, temp_dir.name)
config = load_sq_config(config_file=create_dummy_config_file())
config['data-directory'] = f'{temp_dir.name}/'
# pylint: disable=consider-using-with
tmpfile = NamedTemporaryFile(suffix='.yml', delete=False)
with open(tmpfile.name, 'w') as f:
yaml.dump(config, f)
return temp_dir, tmpfile
def test_rest_server():
'''Try starting the REST server, actually'''
# pylint: disable=import-outside-toplevel
import subprocess
from time import sleep
import requests
cfgfile = create_dummy_config_file(
datadir='./tests/data/multidc/parquet-out')
# pylint: disable=consider-using-with
server = subprocess.Popen(
f'./suzieq/restServer/sq_rest_server.py -c {cfgfile} --no-https'.split(
))
sleep(5)
assert (server.pid)
assert (requests.get('http://localhost:8000/api/docs'))
server.kill()
sleep(5)
# pylint: disable=consider-using-with
server = subprocess.Popen(
f'./suzieq/restServer/sq_rest_server.py -c {cfgfile} '.split())
sleep(5)
assert (server.pid)
assert (requests.get('https://localhost:8000/api/docs', verify=False))
server.kill()
sleep(5)
os.remove(cfgfile)
def manager_cfg():
"""Get a dummy configuration file
"""
args = {}
args['config'] = create_dummy_config_file()
args['config-dict'] = load_sq_config(args['config'])
yield args
os.remove(args['config'])
def test_server_exec():
'''Can we can get a valid response with & without https'''
# Generate a random port
port = randint(9000, 10000)
# We need to change the port used to avoid conflicts
cfgfile = create_dummy_config_file()
sqcfg = load_sq_config(cfgfile)
if 'rest' not in sqcfg:
sqcfg['rest'] = {'port': port}
else:
sqcfg['rest']['port'] = port
with open(cfgfile, 'w') as f:
f.write(yaml.safe_dump(sqcfg))
server_cmd_args = f'{suzieq_rest_server_path} -c {cfgfile}'.split()
# pylint: disable=consider-using-with
proc = subprocess.Popen(server_cmd_args)
# Try a request from the server
sleep(5)
resp = requests.get(f'https://localhost:{port}/api/docs', verify=False)
assert (resp.status_code == 200)
# Try a non-https request from the server
sleep(5)
try:
resp = requests.get(f'http://localhost:{port}/api/docs', verify=False)
assert (resp.status_code != 200)
except requests.exceptions.ConnectionError:
pass
proc.kill()
# Now test without https
server_cmd_args = (
f'{suzieq_rest_server_path} -c {cfgfile} --no-https'.split())
# pylint: disable=consider-using-with
proc = subprocess.Popen(server_cmd_args)
# Try a request from the server
sleep(5)
resp = requests.get(f'http://localhost:{port}/api/docs', verify=False)
assert (resp.status_code == 200)
# Try a https request from the server
sleep(5)
try:
resp = requests.get(f'https://localhost:{port}/api/docs', verify=False)
assert (resp.status_code != 200)
except requests.exceptions.ConnectionError:
pass
proc.kill()
os.remove(cfgfile)
def create_config(testvar):
if 'data-directory' in testvar:
# We need to create a tempfile to hold the config
tf = conftest.create_dummy_config_file()
tmpconfig = load_sq_config(tf)
tmpconfig['data-directory'] = testvar['data-directory']
with open(tf, 'w') as f:
f.write(yaml.dump(tmpconfig))
return tf
def create_config(testvar):
if 'data-directory' in testvar:
# We need to create a tempfile to hold the config
tmpconfig = load_sq_config(conftest.create_dummy_config_file())
tmpconfig['data-directory'] = testvar['data-directory']
tf = tempfile.NamedTemporaryFile(delete=False)
with open(tf.name, 'w') as f:
f.write(yaml.dump(tmpconfig))
return tf.name
def create_config():
# We need to create a tempfile to hold the config
tmpconfig = load_sq_config(config_file=conftest.create_dummy_config_file())
tmpconfig['data-directory'] = './tests/data/multidc/parquet-out'
r_int = random.randint(17, 2073)
fname = f'/tmp/suzieq-cfg-{r_int}.yml'
with open(fname, 'w') as f:
f.write(yaml.dump(tmpconfig))
return fname
def app_initialize():
'''Initialize the test server'''
# pylint: disable=import-outside-toplevel
from suzieq.restServer.query import app_init
cfgfile = create_dummy_config_file(
datadir='./tests/data/multidc/parquet-out')
app_init(cfgfile)
yield
os.remove(cfgfile)
def test_server_exec():
'''We fire up the rest server and see if we can get a valid response'''
cfgfile = create_dummy_config_file()
server_cmd_args = f'{suzieq_rest_server_path} -c {cfgfile}'.split()
proc = subprocess.Popen(server_cmd_args)
# Try a request from the server
sleep(5)
resp = requests.get(f'https://localhost:8000/docs', verify=False)
proc.kill()
assert(resp.status_code == 200)
def create_config(t_dir, suzieq_dir):
# We need to create a tempfile to hold the config
tmpconfig = load_sq_config(conftest.create_dummy_config_file())
tmpconfig['data-directory'] = f"{t_dir}/parquet-out"
tmpconfig[
'service-directory'] = f"{suzieq_dir}/{tmpconfig['service-directory']}"
tmpconfig[
'schema-directory'] = f"{suzieq_dir}/{tmpconfig['schema-directory']}"
fname = f'{t_dir}/suzieq-cfg.yml'
with open(fname, 'w') as f:
f.write(yaml.dump(tmpconfig))
return fname
def generate_controller(args: Dict, inv_file: str = None,
conf_file: str = '') -> Controller:
"""Generate a Controller object
Args:
args (Dict): controller input args
inv_file (str, optional): controller inventory file. Defaults to None.
conf_file (str, optional): controller config file. Defaults to ''.
Returns:
Controller: the newly created Controller
"""
if conf_file == '':
conf_file = create_dummy_config_file()
config = load_sq_config(config_file=conf_file)
args = update_args(args, inv_file, conf_file)
parse_args = generate_argparse(args)
return Controller(parse_args, config)
def test_sqcmds_regex_hostname(table, datadir):
cfgfile = create_dummy_config_file(datadir=datadir)
df = get_sqobject(table)(config_file=cfgfile).get(
hostname=['~leaf.*', '~exit.*'])
if table == 'tables':
if 'junos' in datadir:
assert df[df.table == 'device']['deviceCnt'].tolist() == [4]
elif not any(x in datadir for x in ['vmx', 'mixed']):
# The hostnames for these output don't match the hostname regex
assert df[df.table == 'device']['deviceCnt'].tolist() == [6]
return
if 'basic_dual_bgp' in datadir and table in [
'ospf', 'evpnVni', 'devconfig'
]:
return
if not any(x in datadir for x in ['vmx', 'mixed', 'junos']):
assert not df.empty
if table in ['macs', 'vlan'] and 'basic_dual_bgp' in datadir:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04'])
elif table not in ['mlag']:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04', 'exit01', 'exit02'])
else:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'leaf03', 'leaf04'])
elif 'junos' in datadir:
if table == 'mlag':
# Our current Junos tests don't have MLAG
return
assert not df.empty
if table == 'macs':
assert set(df.hostname.unique()) == set(['leaf01', 'leaf02'])
else:
assert set(df.hostname.unique()) == set(
['leaf01', 'leaf02', 'exit01', 'exit02'])
def poller_args(monkeypatch):
"""Generate a config file and some standard
user arguments
"""
# Set two enviroment variable containing the path to a dummy inventory
# and a dummy key for credential decryption
monkeypatch.setenv('SQ_CONTROLLER_POLLER_CRED', INV_CREDENTIAL_KEY)
monkeypatch.setenv('SQ_INVENTORY_PATH', INVENTORY_PATH)
cfg_file = create_dummy_config_file()
userargs_dict = {
'namespace': None,
'input_dir': None,
'service_only': None,
'exclude_services': None,
'outputs': ['parquet'],
'output_dir': 'tests/unit/poller/worker/poller_output/parquet_out',
'config': cfg_file,
'run_once': None,
'ssh_config_file': None,
'worker_id': '0'
}
userargs = argparse.Namespace(**userargs_dict)
yield userargs
os.remove(cfg_file)
def coalescer_cfg():
"""Generate a random data directory where to start the poller
"""
cfg_file = create_dummy_config_file()
yield cfg_file
os.remove(cfg_file)
def _test_data(topology, proto, scenario, testvar):
# pylint: disable=redefined-outer-name
name = f'{topology}_{proto}_{scenario}'
testvar['data-directory'] = f"{parquet_dir}/{name}/parquet-out"
dummy_config = load_sq_config(conftest.create_dummy_config_file())
_test_sqcmds(dummy_config, testvar)