def test_errors_on_invalid_elements(self, from_config: mock.Mock):
from_config.side_effect = ConfigurationError()
with self.assertRaisesRegex(ConfigurationError, 'element'):
data_stream.from_config(self.base_config)
def validate_name(name: str) -> str:
if name is None or len(name) == 0:
raise ConfigurationError("missing name")
if '/' in name:
raise ConfigurationError("invalid name, '\\' not allowed")
return name
def validate_name(name: str) -> str:
if len(name) == 0:
raise ConfigurationError("missing name")
return name
def run(custom_values=None, verify=True) -> config.JouleConfig:
"""provide a dict INI configuration to override defaults
if verify is True, perform checks on settings to make sure they are appropriate"""
my_configs = configparser.ConfigParser()
my_configs.read_dict(config.DEFAULT_CONFIG)
if custom_values is not None:
my_configs.read_dict(custom_values)
main_config = my_configs['Main']
# Node name
node_name = main_config['Name']
# ModuleDirectory
module_directory = main_config['ModuleDirectory']
if not os.path.isdir(module_directory) and verify:
raise ConfigurationError(
"ModuleDirectory [%s] does not exist" % module_directory)
# StreamDirectory
stream_directory = main_config['StreamDirectory']
if not os.path.isdir(stream_directory) and verify:
raise ConfigurationError(
"StreamDirectory [%s] does not exist" % stream_directory)
# Specify IPAddress and Port to listen on network interface
# IPAddress
if 'IPAddress' in main_config:
ip_address = main_config['IPAddress']
try:
ipaddress.ip_address(ip_address)
except ValueError as e:
raise ConfigurationError("IPAddress is invalid") from e
# Port
try:
port = int(main_config['Port'])
if port < 0 or port > 65535:
raise ValueError()
except ValueError as e:
raise ConfigurationError("Port must be between 0 - 65535") from e
else:
port = None
ip_address = None
# SocketDirectory
socket_directory = main_config['SocketDirectory']
if not os.path.isdir(socket_directory) and verify:
try:
os.mkdir(socket_directory)
# make sure the ownership is correct
os.chmod(socket_directory, 0o700)
except FileExistsError:
raise ConfigurationError("SocketDirectory [%s] is a file" % socket_directory)
except PermissionError:
raise ConfigurationError("Cannot create SocketDirectory at [%s]" % socket_directory)
if not os.access(socket_directory, os.W_OK) and verify:
raise ConfigurationError(
"SocketDirectory [%s] is not writable" % socket_directory)
# Nilmdb URL
if 'NilmdbUrl' in main_config and main_config['NilmdbUrl'] != '':
nilmdb_url = main_config['NilmdbUrl']
if verify:
loop = asyncio.get_event_loop()
loop.run_until_complete(verify_nilmdb_url(nilmdb_url))
else:
nilmdb_url = None
# Database
if 'Database' in main_config:
database = "postgresql://" + main_config['Database']
elif verify:
raise ConfigurationError("Missing [Database] configuration")
else: # pragma: no cover
database = '' # this is invalid of course, just used in unit testing
if verify:
# check to see if this is a valid database DSN
try:
conn = psycopg2.connect(database)
conn.close()
except psycopg2.Error:
raise ConfigurationError("Cannot connect to database [%s]" % database)
# InsertPeriod
try:
insert_period = int(main_config['InsertPeriod'])
if insert_period <= 0:
raise ValueError()
except ValueError:
raise ConfigurationError("InsertPeriod must be a postive number")
# CleanupPeriod
try:
cleanup_period = int(main_config['CleanupPeriod'])
if cleanup_period <= 0 or cleanup_period < insert_period:
raise ValueError()
except ValueError:
raise ConfigurationError("CleanupPeriod must be a postive number > InsertPeriod")
# Max Log Lines
try:
max_log_lines = int(main_config['MaxLogLines'])
if max_log_lines <= 0:
raise ValueError()
except ValueError:
raise ConfigurationError("MaxLogLines must be a postive number")
# Security configuration
if 'Security' in my_configs:
security = config.SecurityConfig(my_configs['Security']['Certificate'],
my_configs['Security']['Key'],
my_configs.get('Security',
'CertificateAuthority', fallback=""))
else:
security = None
# Proxies
uuid = 0
proxies = []
if 'Proxies' in my_configs:
for name in my_configs['Proxies']:
url_str = my_configs['Proxies'][name]
# make sure proxy url ends with /
if url_str[-1] != '/':
url_str += '/'
url = yarl.URL(url_str)
proxies.append(Proxy(name, uuid, url))
uuid += 1
return config.JouleConfig(
name=node_name,
module_directory=module_directory,
stream_directory=stream_directory,
ip_address=ip_address,
port=port,
socket_directory=socket_directory,
database=database,
insert_period=insert_period,
cleanup_period=cleanup_period,
max_log_lines=max_log_lines,
nilmdb_url=nilmdb_url,
proxies=proxies,
security=security
)
def main(argv=None):
parser = argparse.ArgumentParser("Joule Daemon")
parser.add_argument("--config", default="/etc/joule/main.conf")
xargs = parser.parse_args(argv)
log.addFilter(LogDedupFilter())
logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s',
level=logging.WARNING)
if xargs.config is not None:
if os.path.isfile(xargs.config) is False:
log.error("Invalid configuration: cannot load file [%s]" %
xargs.config)
exit(1)
my_config = None
try:
cparser = configparser.ConfigParser()
r = cparser.read(xargs.config)
if len(r) != 1:
raise ConfigurationError(f"cannot read {xargs.config}")
my_config = load_config.run(custom_values=cparser)
except ConfigurationError as e:
log.error("Invalid configuration: %s" % e)
exit(1)
# uvloop uses libuv which does not support
# connections to abstract namespace sockets
# https://github.com/joyent/libuv/issues/1486
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
loop.set_debug(True)
daemon = Daemon(my_config)
try:
daemon.initialize()
except SQLAlchemyError as e:
print("""
Error initializing database, ensure user 'joule' has sufficient permissions:
From a shell run
$> sudo -u postgres psql
postgres=# ALTER ROLE joule WITH CREATEROLE REPLICATION;
postgres=# GRANT ALL PRIVILEGES ON DATABASE joule TO joule WITH GRANT OPTION;
postgres=# GRANT pg_read_all_settings TO joule;
""")
loop.close()
exit(1)
loop.add_signal_handler(signal.SIGINT, daemon.stop)
loop.add_signal_handler(signal.SIGTERM, daemon.stop)
async def debugger():
task_list = []
while True:
for t in asyncio.all_tasks():
name = t.get_name()
if "Task" in name:
if "aiohttp" in str(t.get_stack()):
t.set_name("aiohttp")
elif "StreamReader.read" in str(t.get_coro()):
t.set_name("StreamReader.read")
else:
t.set_name(f"UNK {t.get_coro()}")
if t not in task_list:
task_list.append(t)
# print(f"New Task[{id(t)}]: {t.get_name()}")
for t in task_list:
if t.done():
# print(f"DONE: {t.get_name()}: ", end="")
# if t.cancelled():
# print("cancelled")
if not t.cancelled():
exception = t.exception()
if exception is not None:
print("Got exception", t.exception())
# print("----Cancelling Daemon----")
# daemon_task.cancel()
# else:
# print("completed")
task_list.remove(t)
# else:
# print(f"RUNNING: {t.get_name()}")
await asyncio.sleep(2)
debug = loop.create_task(debugger())
daemon_task = loop.create_task(daemon.run())
daemon_task.set_name("daemon")
debug.set_name("debugger")
loop.run_until_complete(daemon_task)
debug.cancel()
try:
loop.run_until_complete(debug)
except asyncio.CancelledError:
pass
loop.close()
# clear out the socket directory
for file_name in os.listdir(my_config.socket_directory):
path = os.path.join(my_config.socket_directory, file_name)
os.unlink(path)
exit(0)
def _get_bool(setting: str, config: configparser.ConfigParser, default: bool):
try:
return config.getboolean(setting, default)
except ValueError as e:
raise ConfigurationError("[%s] invalid value, use True/False" %
setting) from e
def validate_name(name: str) -> str:
if name == "":
raise ConfigurationError("missing name")
return name