def init_services_default(event_loop):
configs = os.path.abspath(os.curdir) + '/config/'
schema = configs + 'schema/'
mock_queue = Mock()
services = event_loop.run_until_complete(
init_services(configs, schema, mock_queue, True))
return services
async def start_poller(userargs, cfg):
logfile = cfg.get('poller-log-file', '/tmp/sq-poller.log')
loglevel = cfg.get("logging-level", "WARNING")
logger = init_logger('suzieq.poller', logfile, loglevel, False)
if userargs.devices_file and userargs.namespace:
logger.error("Cannot specify both -D and -n options")
sys.exit(1)
if not os.path.exists(cfg["service-directory"]):
logger.error("Service directory {} is not a directory".format(
cfg['service-directory']))
print("Service directory {} is not a directory".format(
cfg['service-directory']))
sys.exit(1)
if userargs.jump_host and not userargs.jump_host.startswith('//'):
logger.error("Jump host format is //<username>@<jumphost>:<port>")
print("Jump host format is //<username>@<jumphost>:<port>")
sys.exit(1)
if not cfg.get("schema-directory", None):
schema_dir = "{}/{}".format(userargs.service_dir, "schema")
else:
schema_dir = cfg["schema-directory"]
if userargs.ssh_config_file:
ssh_config_file = os.path.expanduser(userargs.ssh_config_file)
if (os.stat(os.path.dirname(ssh_config_file)).st_mode | 0o40700 !=
0o40700):
logger.error('ssh directory has wrong permissions, must be 0700')
print('ERROR: ssh directory has wrong permissions, must be 0700')
sys.exit(1)
output_args = {}
if "parquet" in userargs.outputs:
validate_parquet_args(cfg, output_args, logger)
if userargs.run_once:
userargs.outputs = ["gather"]
output_args['output_dir'] = userargs.output_dir
# Disable coalescer in specific, unusual cases
# in case of input_dir, we also seem to leave a coalescer instance running
if userargs.run_once or userargs.input_dir:
userargs.no_coalescer = True
outputs = init_output_workers(userargs.outputs, output_args)
queue = asyncio.Queue()
logger.info("Initializing hosts and services")
tasks = []
ignore_known_hosts = userargs.ignore_known_hosts or None
if userargs.service_only:
svclist = userargs.service_only.split()
else:
svcs = list(Path(cfg["service-directory"]).glob('*.yml'))
svclist = [os.path.basename(x).split('.')[0] for x in svcs]
if userargs.exclude_services:
svclist = list(
filter(lambda x: x not in userargs.exclude_services, svclist))
if not svclist:
print(
f"No correct services specified. Should have been one of {[svc.name for svc in svcs]}"
)
sys.exit(1)
if userargs.input_dir:
tasks.append(init_files(userargs.input_dir))
else:
tasks.append(
init_hosts(inventory=userargs.devices_file,
ans_inventory=userargs.ansible_file,
namespace=userargs.namespace,
passphrase=userargs.passphrase,
jump_host=userargs.jump_host,
jump_host_key_file=userargs.jump_host_key_file,
password=userargs.ask_pass,
ignore_known_hosts=ignore_known_hosts))
period = cfg.get('period', 15)
tasks.append(
init_services(cfg["service-directory"], schema_dir, queue, svclist,
period, userargs.run_once or "forever"))
nodes, svcs = await asyncio.gather(*tasks)
if not nodes or not svcs:
# Logging should've been done by init_nodes/services for details
print('Termminating because no nodes or services found')
sys.exit(0)
node_callq = defaultdict(lambda: defaultdict(dict))
node_callq.update({
x: {
'hostname': nodes[x].hostname,
'postq': nodes[x].post_commands
}
for x in nodes
})
for svc in svcs:
svc.set_nodes(node_callq)
logger.setLevel(logging.INFO)
logger.info("Suzieq Started")
logger.setLevel(loglevel.upper())
loop = asyncio.get_event_loop()
for s in [signal.SIGTERM, signal.SIGINT]:
loop.add_signal_handler(
s, lambda s=s: asyncio.create_task(process_signal(s, loop)))
try:
# The logic below of handling the writer worker task separately is to
# ensure we can terminate properly when all the other tasks have
# finished as in the case of using file input instead of SSH
svc_tasks = [svc.run() for svc in svcs]
tasks = [nodes[node].run() for node in nodes]
tasks += [run_output_worker(queue, outputs, logger)]
tasks += svc_tasks
if not userargs.no_coalescer:
tasks += [
start_and_monitor_coalescer(userargs.config, cfg, logger)
]
while tasks:
try:
done, pending = await asyncio.wait(
tasks, return_when=asyncio.FIRST_COMPLETED)
tasks = list(pending)
if tasks and any(i._coro in svc_tasks for i in tasks):
continue
else:
break
except asyncio.CancelledError:
break
finally:
logger.warning("sq-poller: Received terminate signal. Terminating")
loop.stop()
return