def test_errors_on_invalid_max_log_lines(self):
parser = configparser.ConfigParser()
for bad_val in [-1, 'abc']:
parser.read_string("""
[Main]
MaxLogLines = %s
""" % bad_val)
with self.assertRaisesRegex(ConfigurationError, "MaxLogLines"):
load_config.run(custom_values=parser, verify=False)
def test_error_on_bad_ip_address(self):
parser = configparser.ConfigParser()
bad_ips = ["8.8.x.8", "bad", "", "900.8.4.100"]
for ip in bad_ips:
parser.read_string("""
[Main]
IPAddress=%s
""" % ip)
with self.assertRaisesRegex(ConfigurationError, "IPAddress"):
load_config.run(custom_values=parser, verify=False)
def test_errors_on_invalid_insert_period(self):
bad_periods = ['-1', '0', 'abc', '']
for period in bad_periods:
parser = configparser.ConfigParser()
parser.read_string("""
[Main]
InsertPeriod = %s
""" % period)
with self.assertRaisesRegex(ConfigurationError, "InsertPeriod"):
load_config.run(custom_values=parser, verify=False)
def test_error_on_bad_port(self):
parser = configparser.ConfigParser()
bad_ports = ["-3", "99999", "abc", ""]
for port in bad_ports:
parser.read_string("""
[Main]
IpAddress=127.0.0.1
Port=%s
""" % port)
with self.assertRaisesRegex(ConfigurationError, "Port"):
load_config.run(custom_values=parser, verify=False)
def test_error_if_missing_database_configuration(self):
with tempfile.TemporaryDirectory() as module_dir:
with tempfile.TemporaryDirectory() as stream_dir:
with tempfile.TemporaryDirectory() as sock_dir:
parser = configparser.ConfigParser()
parser.read_string("""
[Main]
ModuleDirectory=%s
StreamDirectory=%s
SocketDirectory=%s
""" % (module_dir, stream_dir, sock_dir))
with self.assertRaisesRegex(ConfigurationError,
"database"):
load_config.run(custom_values=parser)
def test_error_if_directories_do_not_exist(self):
with tempfile.TemporaryDirectory() as good_dir1:
with tempfile.TemporaryDirectory() as good_dir2:
bad_dir = "/does/not/exist"
bad_module_dir = (bad_dir, good_dir1, good_dir2)
bad_stream_dir = (good_dir1, bad_dir, good_dir2)
for setup in zip(bad_module_dir, bad_stream_dir):
parser = configparser.ConfigParser()
parser.read_string("""
[Main]
ModuleDirectory=%s
StreamDirectory=%s
""" % setup)
with self.assertRaises(ConfigurationError):
load_config.run(custom_values=parser)
def admin_erase(config, links):
"""Erase the local node."""
from joule.services import load_config
from joule.errors import ConfigurationError
# make sure joule is not running
pid_file = os.path.join(WORKING_DIRECTORY, 'pid')
if os.path.exists(pid_file):
with open(pid_file, 'r') as f:
pid = int(f.readline())
if psutil.pid_exists(pid):
raise click.ClickException(
"stop joule service before running this command")
# load the config file
if os.path.isfile(config) is False:
raise click.ClickException(
"Invalid configuration: cannot load file [%s]" % config)
parser = configparser.ConfigParser()
try:
with open(config, 'r') as f:
parser.read_file(f)
except PermissionError:
raise click.ClickException("insufficient permissions, run with [sudo]")
except FileNotFoundError:
raise click.ClickException(
"Joule config [%s] not found, specify with --config")
try:
joule_config = load_config.run(custom_values=parser)
except ConfigurationError as e:
raise click.ClickException("Invalid configuration: %s" % e)
loop = asyncio.get_event_loop()
loop.run_until_complete(run(joule_config, links))
loop.close()
def test_errors_if_nilmdb_not_available(self):
with tempfile.TemporaryDirectory() as module_dir:
with tempfile.TemporaryDirectory() as stream_dir:
with tempfile.TemporaryDirectory() as sock_dir:
parser = configparser.ConfigParser()
parser.read_string("""
[Main]
ModuleDirectory=%s
StreamDirectory=%s
SocketDirectory=%s
NilmdbUrl=http://127.0.0.1:234/bad_nilmdb
""" % (module_dir, stream_dir, sock_dir))
with self.assertRaisesRegex(ConfigurationError, "NilmDB"):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
load_config.run(custom_values=parser)
loop.close()
def test_customizes_config(self):
parser = configparser.ConfigParser()
parser.read_string("""
[Main]
IPAddress = 8.8.8.8
Port = 8080
Database = new_db
InsertPeriod = 20
""")
my_config = load_config.run(custom_values=parser, verify=False)
self.assertEqual(my_config.ip_address, "8.8.8.8")
self.assertEqual(my_config.insert_period, 20)
def test_loads_proxies(self):
parser = configparser.ConfigParser()
parser.read_string("""
[Proxies]
site1=http://localhost:5000
site two=https://othersite.com
""")
my_config = load_config.run(custom_values=parser, verify=False)
self.assertEqual(my_config.proxies[0].url,
yarl.URL("http://localhost:5000"))
self.assertEqual(my_config.proxies[0].uuid, 0)
self.assertEqual(my_config.proxies[0].name, "site1")
self.assertEqual(my_config.proxies[1].url,
yarl.URL("https://othersite.com"))
self.assertEqual(my_config.proxies[1].uuid, 1)
self.assertEqual(my_config.proxies[1].name, "site two")
def test_verifies_directories_exist(self):
postgresql = testing.postgresql.Postgresql()
db_url = postgresql.url()
with tempfile.TemporaryDirectory() as module_dir:
with tempfile.TemporaryDirectory() as stream_dir:
with tempfile.TemporaryDirectory() as socket_dir:
parser = configparser.ConfigParser()
parser.read_string(
"""
[Main]
ModuleDirectory=%s
StreamDirectory=%s
SocketDirectory=%s
Database=%s
""" %
(module_dir, stream_dir, socket_dir, db_url[13:]))
my_config = load_config.run(custom_values=parser)
self.assertEqual(my_config.stream_directory, stream_dir)
self.assertEqual(my_config.module_directory, module_dir)
postgresql.stop()
def main(argv=None):
parser = argparse.ArgumentParser("Joule Daemon")
parser.add_argument("--config", default="/etc/joule/main.conf")
xargs = parser.parse_args(argv)
log.addFilter(LogDedupFilter())
logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s',
level=logging.WARNING)
if xargs.config is not None:
if os.path.isfile(xargs.config) is False:
log.error("Invalid configuration: cannot load file [%s]" %
xargs.config)
exit(1)
my_config = None
try:
cparser = configparser.ConfigParser()
r = cparser.read(xargs.config)
if len(r) != 1:
raise ConfigurationError(f"cannot read {xargs.config}")
my_config = load_config.run(custom_values=cparser)
except ConfigurationError as e:
log.error("Invalid configuration: %s" % e)
exit(1)
# uvloop uses libuv which does not support
# connections to abstract namespace sockets
# https://github.com/joyent/libuv/issues/1486
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
loop.set_debug(True)
daemon = Daemon(my_config)
try:
daemon.initialize()
except SQLAlchemyError as e:
print("""
Error initializing database, ensure user 'joule' has sufficient permissions:
From a shell run
$> sudo -u postgres psql
postgres=# ALTER ROLE joule WITH CREATEROLE REPLICATION;
postgres=# GRANT ALL PRIVILEGES ON DATABASE joule TO joule WITH GRANT OPTION;
postgres=# GRANT pg_read_all_settings TO joule;
""")
loop.close()
exit(1)
loop.add_signal_handler(signal.SIGINT, daemon.stop)
loop.add_signal_handler(signal.SIGTERM, daemon.stop)
async def debugger():
task_list = []
while True:
for t in asyncio.all_tasks():
name = t.get_name()
if "Task" in name:
if "aiohttp" in str(t.get_stack()):
t.set_name("aiohttp")
elif "StreamReader.read" in str(t.get_coro()):
t.set_name("StreamReader.read")
else:
t.set_name(f"UNK {t.get_coro()}")
if t not in task_list:
task_list.append(t)
# print(f"New Task[{id(t)}]: {t.get_name()}")
for t in task_list:
if t.done():
# print(f"DONE: {t.get_name()}: ", end="")
# if t.cancelled():
# print("cancelled")
if not t.cancelled():
exception = t.exception()
if exception is not None:
print("Got exception", t.exception())
# print("----Cancelling Daemon----")
# daemon_task.cancel()
# else:
# print("completed")
task_list.remove(t)
# else:
# print(f"RUNNING: {t.get_name()}")
await asyncio.sleep(2)
debug = loop.create_task(debugger())
daemon_task = loop.create_task(daemon.run())
daemon_task.set_name("daemon")
debug.set_name("debugger")
loop.run_until_complete(daemon_task)
debug.cancel()
try:
loop.run_until_complete(debug)
except asyncio.CancelledError:
pass
loop.close()
# clear out the socket directory
for file_name in os.listdir(my_config.socket_directory):
path = os.path.join(my_config.socket_directory, file_name)
os.unlink(path)
exit(0)
def admin_ingest(config, backup, node, map, pgctl_binary, yes, start, end):
"""Restore data from a backup."""
# expensive imports so only execute if the function is called
from joule.services import load_config
import sqlalchemy
from sqlalchemy.orm import Session
from joule.models import Base, TimescaleStore, NilmdbStore
parser = configparser.ConfigParser()
loop = asyncio.get_event_loop()
# make sure either a backup or a node is specified
if (((backup is None) and (node is None))
or ((backup is not None) and (node is not None))):
raise click.ClickException(
"Specify either a backup or a node to ingest data from")
# make sure the time bounds make sense
if start is not None:
try:
start = utilities.human_to_timestamp(start)
except ValueError:
raise errors.ApiError("invalid start time: [%s]" % start)
if end is not None:
try:
end = utilities.human_to_timestamp(end)
except ValueError:
raise errors.ApiError("invalid end time: [%s]" % end)
if (start is not None) and (end is not None) and ((end - start) <= 0):
raise click.ClickException(
"Error: start [%s] must be before end [%s]" %
(utilities.timestamp_to_human(start),
utilities.timestamp_to_human(end)))
# parse the map file if specified
stream_map = None
if map is not None:
stream_map = []
try:
with open(map, newline='') as csvfile:
reader = csv.reader(csvfile,
delimiter=',',
quotechar='|',
skipinitialspace=True)
for row in reader:
if len(row) == 0: # ignore blank lines
continue
if len(row) == 1 and len(
row[0]) == 0: # line with only whitespace
continue
if row[0][0] == '#': # ignore comments
continue
if len(row) != 2:
raise errors.ConfigurationError(
"""invalid map format. Refer to template below:
# this line is a comment
# only paths in this file will be copied
# source and destination paths are separated by a ','
/source/path, /destination/path
/source/path2, /destination/path2
#..etc
""")
stream_map.append(row)
except FileNotFoundError:
raise click.ClickException("Cannot find map file at [%s]" % map)
except PermissionError:
raise click.ClickException("Cannot read map file at [%s]" % map)
except errors.ConfigurationError as e:
raise click.ClickException(str(e))
# load the Joule configuration file
try:
with open(config, 'r') as f:
parser.read_file(f, config)
joule_config = load_config.run(custom_values=parser)
except FileNotFoundError:
raise click.ClickException(
"Cannot load joule configuration file at [%s]" % config)
except PermissionError:
raise click.ClickException(
"Cannot read joule configuration file at [%s] (run as root)" %
config)
except errors.ConfigurationError as e:
raise click.ClickException("Invalid configuration: %s" % e)
dest_engine = sqlalchemy.create_engine(joule_config.database)
Base.metadata.create_all(dest_engine)
dest_db = Session(bind=dest_engine)
if joule_config.nilmdb_url is not None:
dest_datastore = NilmdbStore(joule_config.nilmdb_url, 0, 0, loop)
else:
dest_datastore = TimescaleStore(joule_config.database, 0, 0, loop)
# demote priveleges
if "SUDO_GID" in os.environ:
os.setgid(int(os.environ["SUDO_GID"]))
if "SUDO_UID" in os.environ:
os.setuid(int(os.environ["SUDO_UID"]))
# create a log file for exec cmds
pg_log_name = "joule_restore_log_%s.txt" % uuid.uuid4().hex.upper()[0:6]
pg_log = open(pg_log_name, 'w')
# if pgctl_binary is not specified, try to autodect it
if pgctl_binary is None:
try:
completed_proc = subprocess.run(["psql", "-V"],
stdout=subprocess.PIPE)
output = completed_proc.stdout.decode('utf-8')
version = output.split(" ")[2]
major_version = version.split(".")[0]
pgctl_binary = "/usr/lib/postgresql/%s/bin/pg_ctl" % major_version
except (FileNotFoundError, IndexError):
raise click.ClickException(
"cannot autodetect pg_ctl location, specify with -b")
# determine if the source is a backup or a node
if node is not None:
live_restore = True
src_dsn = loop.run_until_complete(get_dsn(node))
# check whether the source uses nilmdb
click.echo("WARNING: Nilmdb sources are not supported yet")
src_datastore = TimescaleStore(src_dsn, 0, 0, loop)
nilmdb_proc = None
else:
if not os.path.isdir(backup):
raise click.ClickException("backup [%s] does not exist" % backup)
src_dsn = start_src_db(backup, pgctl_binary, pg_log)
# check whether the source uses nilmdb
nilmdb_path = os.path.join(backup, 'nilmdb')
nilmdb_proc = None
if os.path.exists(nilmdb_path):
port = unused_port()
nilmdb_proc = start_src_nilmdb(nilmdb_path, port, pg_log)
click.echo("waiting for nilmdb to initialize...")
time.sleep(2)
src_datastore = NilmdbStore('http://127.0.0.1:%d' % port, 0, 0,
loop)
if joule_config.nilmdb_url is None:
click.echo(
"Note: re-copying from NilmDB to Timescale may result in --nothing to copy-- messages"
)
else:
src_datastore = TimescaleStore(src_dsn, 0, 0, loop)
live_restore = False
src_engine = sqlalchemy.create_engine(src_dsn)
num_tries = 0
max_tries = 1
while True:
try:
Base.metadata.create_all(src_engine)
break
except sqlalchemy.exc.OperationalError as e:
if live_restore:
raise click.ClickException(
str(e)) # this should work immediately
num_tries += 1
click.echo("... attempting to connect to source database (%d/%d)" %
(num_tries, max_tries))
time.sleep(2)
if num_tries >= max_tries:
raise click.ClickException(
"cannot connect to source database, log saved in [%s]" %
pg_log_name)
src_db = Session(bind=src_engine)
try:
loop.run_until_complete(
run(src_db, dest_db, src_datastore, dest_datastore, stream_map,
yes, start, end))
except errors.ConfigurationError as e:
print("Logs written to [%s]" % pg_log_name)
raise click.ClickException(str(e))
finally:
# close connections
dest_db.close()
src_db.close()
loop.run_until_complete(dest_datastore.close())
loop.run_until_complete(src_datastore.close())
# clean up database if not a live_restore
if not live_restore:
args = ["-D", os.path.join(backup)]
args += ["stop"]
cmd = [pgctl_binary] + args
subprocess.call(cmd, stderr=pg_log, stdout=pg_log)
sock_path = os.path.join(backup, 'sock')
sockets = os.listdir(sock_path)
for s in sockets:
os.remove(os.path.join(sock_path, s))
os.rmdir(sock_path)
if nilmdb_proc is not None:
nilmdb_proc.terminate()
nilmdb_proc.communicate()
pg_log.close()
os.remove(pg_log_name)
click.echo("OK")
def admin_backup(config, folder):
"""Archive entire contents of the local node."""
# expensive imports so only execute if the function is called
from joule.services import load_config
parser = configparser.ConfigParser()
# load the Joule configuration file
try:
with open(config, 'r') as f:
parser.read_file(f, config)
config = load_config.run(custom_values=parser)
except FileNotFoundError:
raise click.ClickException(
"Cannot load joule configuration file at [%s]" % config)
except PermissionError:
raise click.ClickException(
"Cannot read joule configuration file at [%s] (run as root)" %
config)
except errors.ConfigurationError as e:
raise click.ClickException("Invalid configuration: %s" % e)
# demote priveleges
if "SUDO_GID" in os.environ:
os.setgid(int(os.environ["SUDO_GID"]))
if "SUDO_UID" in os.environ:
os.setuid(int(os.environ["SUDO_UID"]))
if folder == "joule_backup_NODE_DATE":
folder = "joule_backup_%s_%s" % (
config.name, datetime.now().strftime("%Y%m%d_%H%M"))
if os.path.exists(folder):
raise click.ClickException("Requested folder [%s] already exists" %
folder)
os.mkdir(folder)
# parse the dsn string
parts = dsnparse.parse(config.database)
# backup the database
args = ["--format", "plain"]
args += ["--checkpoint", "fast"]
args += ["--pgdata", folder]
args += ["--wal-method", "stream"]
args += ["--progress"]
args += ["--host", parts.host]
args += ["--port", "%s" % parts.port]
args += ["--username", parts.user]
args += ["--label", "joule_backup"]
cmd = ["pg_basebackup"] + args
pg_proc_env = os.environ.copy()
pg_proc_env["PGPASSWORD"] = parts.password
click.echo("Copying up postgres database...")
subprocess.call(cmd, env=pg_proc_env)
# add the database name and user
db_info = {
"database": parts.database,
"user": parts.user,
"password": parts.secret,
"nilmdb": config.nilmdb_url is not None
}
with open(os.path.join(folder, "info.json"), 'w') as f:
f.write(json.dumps(db_info, indent=2))
if config.nilmdb_url is None:
click.echo("OK")
return
click.echo("Copying nilmdb database...")
# retrieve the nilmdb data folder
loop = asyncio.get_event_loop()
nilmdb_folder = loop.run_until_complete(get_nilmdb_dir(config.nilmdb_url))
nilmdb_backup = os.path.join(folder, "nilmdb")
shutil.copytree(nilmdb_folder, nilmdb_backup, ignore=print_progress)
click.echo("\nOK")
def admin_authorize(config, url):
"""Grant a local user CLI access."""
# expensive imports so only execute if the function is called
from joule.services import load_config
from joule.models import (Base, master)
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
parser = configparser.ConfigParser()
# load the Joule configuration file
try:
with open(config, 'r') as f:
parser.read_file(f, config)
config = load_config.run(custom_values=parser)
except FileNotFoundError:
raise click.ClickException(
"Cannot load joule configuration file at [%s]" % config)
except PermissionError:
raise click.ClickException(
"Cannot read joule configuration file at [%s] (run as root)" %
config)
except errors.ConfigurationError as e:
raise click.ClickException("Invalid configuration: %s" % e)
# create a connection to the database
engine = create_engine(config.database)
with engine.connect() as conn:
conn.execute('CREATE SCHEMA IF NOT EXISTS data')
conn.execute('CREATE SCHEMA IF NOT EXISTS metadata')
Base.metadata.create_all(engine)
db = Session(bind=engine)
if 'SUDO_USER' in os.environ:
username = os.environ["SUDO_USER"]
else:
username = os.environ["LOGNAME"]
try:
nodes = api.get_nodes()
except ValueError as e:
raise click.ClickException(str(e))
# check if this name is associated with a master entry
my_master = db.query(master.Master). \
filter(master.Master.type == master.Master.TYPE.USER). \
filter(master.Master.name == username).first()
if my_master is None:
# create a new master entry
my_master = master.Master()
my_master.key = master.make_key()
my_master.type = master.Master.TYPE.USER
my_master.name = username
db.add(my_master)
# if a url is specified use it
if url is not None:
joule_url = url
# if the Joule server is not hosting a TCP server use the
# default proxy address
elif config.ip_address is None:
joule_url = "https://localhost/joule"
# otherwise use the the server information in the config file
else:
if config.security is not None and config.security.cafile != "":
addr = config.name
elif config.ip_address != "0.0.0.0":
addr = config.ip_address
else:
addr = "127.0.0.1"
if config.security is None:
scheme = "http"
else:
scheme = "https"
joule_url = "%s://%s:%d" % (scheme, addr, config.port)
my_node = api.create_tcp_node(joule_url, my_master.key, config.name)
api.save_node(my_node)
db.commit()
db.close()
click.echo("Access to node [%s] granted to user [%s]" %
(config.name, username))
def test_loads_default_config(self):
my_config = load_config.run(verify=False)
self.assertEqual(my_config.socket_directory,
config.DEFAULT_CONFIG['Main']['SocketDirectory'])
