def cli_node_files(name, environment, system_folders):
""" Prints location important files.
If the specified configuration cannot be found, it exits. Otherwise
it returns the absolute path to the output.
"""
# select configuration name if none supplied
name, environment = (name, environment) if name else \
select_configuration_questionaire("node", system_folders)
# raise error if config could not be found
if not NodeContext.config_exists(name, environment, system_folders):
error(f"The configuration {Fore.RED}{name}{Style.RESET_ALL} with "
f"environment {Fore.RED}{environment}{Style.RESET_ALL} could "
f"not be found.")
exit(1)
# create node context
ctx = NodeContext(name,
environment=environment,
system_folders=system_folders)
# return path of the configuration
info(f"Configuration file = {ctx.config_file}")
info(f"Log file = {ctx.log_file}")
info(f"data folders = {ctx.data_dir}")
info(f"Database labels and files")
for label, path in ctx.databases.items():
info(f" - {label:15} = {path}")
def cli_node_new_configuration(name, environment, system_folders):
"""Create a new configation file.
Checks if the configuration already exists. If this is not the case
a questionaire is invoked to create a new configuration file.
"""
# select configuration name if none supplied
if not name:
name = q.text("Please enter a configuration-name:").ask()
name_new = name.replace(" ", "-")
if name != name_new:
info(f"Replaced spaces from configuration name: {name}")
name = name_new
if not environment:
environment = q.select(
"Please select the environment you want to configure:",
["application", "prod", "acc", "test", "dev"]).ask()
# Check that we can write in this folder
if not check_config_write_permissions(system_folders):
error("Your user does not have write access to all folders. Exiting")
exit(1)
# check that this config does not exist
if NodeContext.config_exists(name, environment, system_folders):
raise FileExistsError(f"Configuration {name} and environment"
f"{environment} already exists!")
# create config in ctx location
cfg_file = configuration_wizard("node", name, environment, system_folders)
info(f"New configuration created: {Fore.GREEN}{cfg_file}{Style.RESET_ALL}")
def cli_node_attach(name, system_folders):
"""Attach the logs from the docker container to the terminal."""
client = docker.from_env()
check_if_docker_deamon_is_running(client)
running_nodes = client.containers.list(
filters={"label": f"{APPNAME}-type=node"})
running_node_names = [node.name for node in running_nodes]
if not name:
name = q.select("Select the node you wish to inspect:",
choices=running_node_names).ask()
else:
post_fix = "system" if system_folders else "user"
name = f"{APPNAME}-{name}-{post_fix}"
if name in running_node_names:
container = client.containers.get(name)
logs = container.attach(stream=True, logs=True)
Thread(target=print_log_worker, args=(logs, ), daemon=True).start()
while True:
try:
time.sleep(1)
except KeyboardInterrupt:
info("Closing log file. Keyboard Interrupt.")
exit(0)
else:
error(f"{Fore.RED}{name}{Style.RESET_ALL} was not running!?")
def cli_node_stop(name, system_folders, all_nodes):
"""Stop a running container. """
client = docker.from_env()
check_if_docker_deamon_is_running(client)
running_nodes = client.containers.list(
filters={"label": f"{APPNAME}-type=node"})
if not running_nodes:
warning("No nodes are currently running.")
return
running_node_names = [node.name for node in running_nodes]
if all_nodes:
for name in running_node_names:
container = client.containers.get(name)
container.kill()
info(f"Stopped the {Fore.GREEN}{name}{Style.RESET_ALL} Node.")
else:
if not name:
name = q.select("Select the node you wish to stop:",
choices=running_node_names).ask()
else:
post_fix = "system" if system_folders else "user"
name = f"{APPNAME}-{name}-{post_fix}"
if name in running_node_names:
container = client.containers.get(name)
container.kill()
info(f"Stopped the {Fore.GREEN}{name}{Style.RESET_ALL} Node.")
else:
error(f"{Fore.RED}{name}{Style.RESET_ALL} is not running?")
def cli_node_clean():
""" This command erases docker volumes"""
client = docker.from_env()
check_if_docker_deamon_is_running(client)
# retrieve all volumes
volumes = client.volumes.list()
canditates = []
msg = "This would remove the following volumes: "
for volume in volumes:
if volume.name[-6:] == "tmpvol":
canditates.append(volume)
msg += volume.name + ","
info(msg)
confirm = q.confirm(f"Are you sure?")
if confirm.ask():
for volume in canditates:
try:
volume.remove()
# info(volume.name)
except docker.errors.APIError as e:
error(f"Failed to remove volume {Fore.RED}'{volume.name}'"
f"{Style.RESET_ALL}. Is it still in use?")
debug(e)
exit(1)
info("Done!")
def update_build(build: int):
files = Path(".").rglob("__build__")
info(f"Updating build number to {build}")
for file in files:
info(f'File: {file}')
with open(file, 'w') as f:
f.write(build)
def wait_and_collect_results(client, task_id):
task = client.get_task(task_id)
while not task.get("complete"):
task = client.get_task(task_id)
info("Waiting for results")
time.sleep(1)
info("Obtaining results")
results = client.get_results(task_id=task.get("id"))
return results
def cli_server_start(ctx, ip, port, debug):
"""Start the server."""
info("Starting server.")
# Run the server
ip = ip or ctx.config['ip'] or '127.0.0.1'
port = port or int(ctx.config['port']) or 5000
info(f"ip: {ip}, port: {port}")
app = ServerApp(ctx)
app.run(ip, port, debug=debug)
def cli_server_import(ctx, file_, drop_all):
""" Import organizations/collaborations/users and tasks.
Especially usefull for testing purposes.
"""
info("Reading yaml file.")
with open(file_) as f:
entities = yaml.safe_load(f.read())
info("Adding entities to database.")
fixture.load(entities, drop_all=drop_all)
def cli_server_start(ctx, ip, port, debug):
"""Start the server."""
info("Starting server.")
# Load the flask.Resources
server.init_resources(ctx)
# Run the server
ip = ip or ctx.config['ip'] or '127.0.0.1'
port = port or int(ctx.config['port']) or 5000
info(f"ip: {ip}, port: {port}")
server.run(ctx, ip, port, debug=debug)
def create_client_and_authenticate(ctx):
"""Create a client and authenticate."""
host = ctx.config['server_url']
port = ctx.config['port']
api_path = ctx.config['api_path']
info(f"Connecting to server at '{host}:{port}{api_path}'")
username = q.text("Username:"******"Password:"******"Could not authenticate with server!")
debug(e)
exit(1)
return client
def cli_server_new(name, environment, system_folders):
"""Create new configuration."""
if not name:
name = q.text("Please enter a configuration-name:").ask()
name_new = name.replace(" ", "-")
if name != name_new:
info(f"Replaced spaces from configuration name: {name}")
name = name_new
# Check that we can write in this folder
if not check_config_write_permissions(system_folders):
error("Your user does not have write access to all folders. Exiting")
exit(1)
# check that this config does not exist
try:
if ServerContext.config_exists(name, environment, system_folders):
error(f"Configuration {Fore.RED}{name}{Style.RESET_ALL} with "
f"environment {Fore.RED}{environment}{Style.RESET_ALL} "
f"already exists!")
exit(1)
except Exception as e:
print(e)
exit(1)
# create config in ctx location
cfg_file = configuration_wizard(name, environment, system_folders)
info(f"New configuration created: {Fore.GREEN}{cfg_file}{Style.RESET_ALL}")
# info(f"root user created.")
info(f"You can start the server by "
f"{Fore.GREEN}vserver start{Style.RESET_ALL}.")
def set_version(spec, version, build, post):
if spec:
update_version_spec(spec)
info("Version specs updated")
if version:
update_version(version)
info("Vesion numbers updated")
if build:
update_build(build)
info("Build number updated")
if post:
update_post(post)
info("Post release version set")
def update_post(post):
files = [file_ for file_ in Path(".").rglob("_version.py") if \
'build' not in str(file_)]
info(f"Setting post-release version to: {post}")
for file_ in files:
info(f'File: {file_}')
with open(file_, 'r') as f:
content = f.read()
new_content = re.sub(pattern, r"\1\2\3\4\5\g<6>{}\8".format(post),
content)
info(f'Writing to file')
with open(file_, 'w') as f:
f.write(new_content)
def update_version_spec(spec: str) -> None:
assert spec in ('final', 'beta', 'alpha', 'candidate')
files = [file_ for file_ in Path(".").rglob("_version.py") if \
'build' not in str(file_)]
for file in files:
info(f'File: {file}')
info(f'Updating spec to: {spec}')
with open(file, 'r') as f:
content = f.read()
new_content = re.sub(pattern,
r"\1\2\g<3>'{}'\5\6\7\8".format(spec),
content)
info(f'Writing to file')
with open(file, 'w') as f:
f.write(new_content)
def update_version(version: str) -> None:
assert re.match(r"\d+.\d+.\d+", version)
files = [file_ for file_ in Path(".").rglob("_version.py") if \
'build' not in str(file_)]
for file in files:
info(f'File: {file}')
info(f"Updating version to {version}")
major, minor, patch = version.split(".")
with open(file, 'r') as f:
content = f.read()
new_content = re.sub(
pattern,
r"\1 {}, {}, {},\3\4\5\6\7\8".format(major, minor,
patch), content)
info(f'Writing to file')
with open(file, 'w') as f:
f.write(new_content)
def cli_node_create_private_key(name, environment, system_folders, upload,
organization_name, overwrite):
"""Create and upload a new private key (use with caughtion)"""
# retrieve context
name, environment = (name, environment) if name else \
select_configuration_questionaire("node", system_folders)
# raise error if config could not be found
NodeContext.LOGGING_ENABLED = False
if not NodeContext.config_exists(name, environment, system_folders):
error(f"The configuration {Fore.RED}{name}{Style.RESET_ALL} with "
f"environment {Fore.RED}{environment}{Style.RESET_ALL} could "
f"not be found.")
exit(1)
# Create node context
ctx = NodeContext(name, environment, system_folders)
# Authenticate with the server to obtain organization name if it wasn't
# provided
if organization_name is None:
client = create_client_and_authenticate(ctx)
organization_name = client.whoami.organization_name
# generate new key, and save it
filename = f"privkey_{organization_name}.pem"
file_ = ctx.type_data_folder(system_folders) / filename
if file_.exists():
warning(f"File '{Fore.CYAN}{file_}{Style.RESET_ALL}' exists!")
if overwrite:
warning("'--override' specified, so it will be overwritten ...")
if file_.exists() and not overwrite:
error("Could not create private key!")
warning(f"If you're **sure** you want to create a new key, "
f"please run this command with the '--overwrite' flag")
warning("Continuing with existing key instead!")
private_key = RSACryptor(file_).private_key
else:
try:
info("Generating new private key")
private_key = RSACryptor.create_new_rsa_key(file_)
except Exception as e:
error(f"Could not create new private key '{file_}'!?")
debug(e)
info("Bailing out ...")
exit(1)
warning(f"Private key written to '{file_}'")
warning(
"If you're running multiple nodes, be sure to copy the private "
"key to the appropriate directories!")
# create public key
info("Deriving public key")
public_key = RSACryptor.create_public_key_bytes(private_key)
# update config file
info("Updating configuration")
ctx.config["encryption"]["private_key"] = str(file_)
ctx.config_manager.put(environment, ctx.config)
ctx.config_manager.save(ctx.config_file)
# upload key to the server
if upload:
info("Uploading public key to the server. "
"This will overwrite any previously existing key!")
if 'client' not in locals():
client = create_client_and_authenticate(ctx)
try:
client.request(f"/organization/{client.whoami.organization_id}",
method="patch",
json={"public_key": bytes_to_base64s(public_key)})
except Exception as e:
error("Could not upload the public key!")
debug(e)
exit(1)
else:
warning("Public key not uploaded!")
info("[Done]")
def cli_server_start(ctx, ip, port, debug, image, keep):
"""Start the server."""
info("Starting server...")
info("Finding Docker daemon.")
docker_client = docker.from_env()
# will print an error if not
check_if_docker_deamon_is_running(docker_client)
# check that this server is not already running
running_servers = docker_client.containers.list(
filters={"label": f"{APPNAME}-type=server"})
for server in running_servers:
if server.name == f"{APPNAME}-{ctx.name}-{ctx.scope}-server":
error(f"Server {Fore.RED}{ctx.name}{Style.RESET_ALL} "
"is already running")
exit(1)
# pull the server docker image
if image is None:
image = ctx.config.get(
"image", "harbor.vantage6.ai/infrastructure/server:latest")
info(f"Pulling latest server image '{image}'.")
try:
pull_if_newer(image)
# docker_client.images.pull(image)
except Exception:
warning("... alas, no dice!")
else:
info(" ... succes!")
info("Creating mounts")
mounts = [
docker.types.Mount("/mnt/config.yaml",
str(ctx.config_file),
type="bind")
]
# FIXME: code duplication with cli_server_import()
# try to mount database
uri = ctx.config['uri']
url = make_url(uri)
environment_vars = None
# If host is None, we're dealing with a file-based DB, like SQLite
if (url.host is None):
db_path = url.database
if not os.path.isabs(db_path):
# We're dealing with a relative path here -> make it absolute
db_path = ctx.data_dir / url.database
basename = os.path.basename(db_path)
dirname = os.path.dirname(db_path)
os.makedirs(dirname, exist_ok=True)
# we're mounting the entire folder that contains the database
mounts.append(
docker.types.Mount("/mnt/database/", dirname, type="bind"))
environment_vars = {
"VANTAGE6_DB_URI": f"sqlite:////mnt/database/{basename}"
}
else:
warning(f"Database could not be transfered, make sure {url.host} "
"is reachable from the Docker container")
info("Consider using the docker-compose method to start a server")
ip_ = f"--ip {ip}" if ip else ""
port_ = f"--port {port}" if port else ""
cmd = f'vserver-local start -c /mnt/config.yaml -e {ctx.environment} ' \
f'{ip_} {port_}'
info(cmd)
info("Run Docker container")
port_ = str(port or ctx.config["port"] or 5000)
container = docker_client.containers.run(
image,
command=cmd,
mounts=mounts,
detach=True,
labels={
f"{APPNAME}-type": "server",
"name": ctx.config_file_name
},
environment=environment_vars,
ports={f"{port_}/tcp": ("127.0.0.1", port_)},
name=ctx.docker_container_name,
auto_remove=not keep,
tty=True)
info(f"Succes! container id = {container}")
def cli_node_start(name, config, environment, system_folders, image, keep,
mount_src):
"""Start the node instance.
If no name or config is specified the default.yaml configuation is
used. In case the configuration file not excists, a questionaire is
invoked to create one. Note that in this case it is not possible to
specify specific environments for the configuration (e.g. test,
prod, acc).
"""
info("Starting node...")
info("Finding Docker deamon")
docker_client = docker.from_env()
check_if_docker_deamon_is_running(docker_client)
if config:
name = Path(config).stem
ctx = NodeContext(name, environment, system_folders, config)
else:
# in case no name is supplied, ask the user to select one
if not name:
name, environment = select_configuration_questionaire(
"node", system_folders)
# check that config exists, if not a questionaire will be invoked
if not NodeContext.config_exists(name, environment, system_folders):
question = f"Configuration '{name}' using environment"
question += f" '{environment}' does not exist.\n Do you want to"
question += f" create this config now?"
if q.confirm(question).ask():
configuration_wizard("node", name, environment, system_folders)
else:
error("Config file couldn't be loaded")
sys.exit(0)
NodeContext.LOGGING_ENABLED = False
ctx = NodeContext(name, environment, system_folders)
# check that this node is not already running
running_nodes = docker_client.containers.list(
filters={"label": f"{APPNAME}-type=node"})
suffix = "system" if system_folders else "user"
for node in running_nodes:
if node.name == f"{APPNAME}-{name}-{suffix}":
error(f"Node {Fore.RED}{name}{Style.RESET_ALL} is already running")
exit(1)
# make sure the (host)-task and -log dir exists
info("Checking that data and log dirs exist")
ctx.data_dir.mkdir(parents=True, exist_ok=True)
ctx.log_dir.mkdir(parents=True, exist_ok=True)
if image is None:
image = ctx.config.get(
"image", "harbor.vantage6.ai/infrastructure/node:latest")
info(f"Pulling latest node image '{image}'")
try:
# docker_client.images.pull(image)
pull_if_newer(image)
except Exception:
warning(' ... alas, no dice!')
else:
info(" ... success!")
info("Creating Docker data volume")
data_volume = docker_client.volumes.create(
f"{ctx.docker_container_name}-vol")
info("Creating file & folder mounts")
# FIXME: should only mount /mnt/database.csv if it is a file!
# FIXME: should obtain mount points from DockerNodeContext
mounts = [
# (target, source)
("/mnt/database.csv", str(ctx.databases["default"])),
("/mnt/log", str(ctx.log_dir)),
("/mnt/data", data_volume.name),
("/mnt/config", str(ctx.config_dir)),
("/var/run/docker.sock", "/var/run/docker.sock"),
]
if mount_src:
# If mount_src is a relative path, docker willl consider it a volume.
mount_src = os.path.abspath(mount_src)
mounts.append(('/vantage6/vantage6-node', mount_src))
# FIXME: Code duplication: Node.__init__() (vantage6/node/__init__.py)
# uses a lot of the same logic. Suggest moving this to
# ctx.get_private_key()
filename = ctx.config.get("encryption", {}).get("private_key")
# filename may be set to an empty string
if not filename:
filename = 'private_key.pem'
# Location may be overridden by the environment
filename = os.environ.get('PRIVATE_KEY', filename)
# If ctx.get_data_file() receives an absolute path, it is returned as-is
fullpath = Path(ctx.get_data_file(filename))
if fullpath:
if Path(fullpath).exists():
mounts.append(("/mnt/private_key.pem", str(fullpath)))
else:
warning(f"private key file provided {fullpath}, "
"but does not exists")
volumes = {}
for mount in mounts:
volumes[mount[1]] = {'bind': mount[0], 'mode': 'rw'}
# Be careful not to use 'environment' as it would override the function
# argument ;-).
env = {
"DATA_VOLUME_NAME": data_volume.name,
"DATABASE_URI": "/mnt/database.csv",
"PRIVATE_KEY": "/mnt/private_key.pem"
}
cmd = f'vnode-local start -c /mnt/config/{name}.yaml -n {name} -e '\
f'{environment} --dockerized'
info(f"Runing Docker container")
# debug(f" with command: '{cmd}'")
# debug(f" with mounts: {volumes}")
# debug(f" with environment: {env}")
container = docker_client.containers.run(image,
command=cmd,
volumes=volumes,
detach=True,
labels={
f"{APPNAME}-type": "node",
"system": str(system_folders),
"name": ctx.config_file_name
},
environment=env,
name=ctx.docker_container_name,
auto_remove=not keep,
tty=True)
info(f"Success! container id = {container}")
def cli_server_import(ctx, file_, drop_all, image, keep):
""" Import organizations/collaborations/users and tasks.
Especially usefull for testing purposes.
"""
info("Starting server...")
info("Finding Docker daemon.")
docker_client = docker.from_env()
# will print an error if not
check_if_docker_deamon_is_running(docker_client)
# pull lastest Docker image
if image is None:
image = ctx.config.get(
"image", "harbor.vantage6.ai/infrastructure/server:latest")
info(f"Pulling latest server image '{image}'.")
try:
docker_client.images.pull(image)
except Exception:
warning("... alas, no dice!")
else:
info(" ... succes!")
info("Creating mounts")
mounts = [
docker.types.Mount("/mnt/config.yaml",
str(ctx.config_file),
type="bind"),
docker.types.Mount("/mnt/import.yaml", str(file_), type="bind")
]
# FIXME: code duplication with cli_server_start()
# try to mount database
uri = ctx.config['uri']
url = make_url(uri)
environment_vars = None
# If host is None, we're dealing with a file-based DB, like SQLite
if (url.host is None):
db_path = url.database
if not os.path.isabs(db_path):
# We're dealing with a relative path here -> make it absolute
db_path = ctx.data_dir / url.database
basename = os.path.basename(db_path)
dirname = os.path.dirname(db_path)
os.makedirs(dirname, exist_ok=True)
# we're mounting the entire folder that contains the database
mounts.append(
docker.types.Mount("/mnt/database/", dirname, type="bind"))
environment_vars = {
"VANTAGE6_DB_URI": f"sqlite:////mnt/database/{basename}"
}
else:
warning(f"Database could not be transfered, make sure {url.host} "
"is reachable from the Docker container")
info("Consider using the docker-compose method to start a server")
drop_all_ = "--drop-all" if drop_all else ""
cmd = f'vserver-local import -c /mnt/config.yaml -e {ctx.environment} ' \
f'{drop_all_} /mnt/import.yaml'
info(cmd)
info("Run Docker container")
container = docker_client.containers.run(image,
command=cmd,
mounts=mounts,
detach=True,
labels={
f"{APPNAME}-type": "server",
"name": ctx.config_file_name
},
environment=environment_vars,
auto_remove=not keep,
tty=True)
logs = container.logs(stream=True, stdout=True)
Thread(target=print_log_worker, args=(logs, ), daemon=False).start()
info(f"Succes! container id = {container.id}")
async def before_printer(self):
info('Signing in to Discord...')
await self.bot.wait_until_ready()
async def update_community(self):
info("Ready to send message")
for channel in self.bot.get_all_channels():
if channel.name == 'general':
await channel.send(embed=self.create_embed(*self.info))
exit()
def master(client: Client, data, incidence, population, gender, ageclass,
prefacture, standard_population, organization_ids=None):
"""Master algoritm.
The master algorithm is the chair of the Round Robin, which makes
sure everyone waits for their turn to identify themselfs.
"""
# get all organizations (ids) that are within the collaboration
# FlaskIO knows the collaboration to which the container belongs
# as this is encoded in the JWT (Bearer token)
if not organization_ids:
organizations = client.get_organizations_in_my_collaboration()
organization_ids = [organization.get("id") for organization in
organizations]
# all participating paties compute crude rate and incidence population
info("Dispatching preliminairy_results")
task_preliminairy_results = client.create_new_task(
input_={
"method": "preliminairy_results",
"kwargs": {
"incidence": incidence,
"population": population,
"gender": gender,
"ageclass": ageclass,
"prefacture": prefacture
}
},
organization_ids=organization_ids
)
# while we wait, lets compute the relative population dataframe
info('Calculating relative population')
rel_pop = relative_population(standard_population,
population, ageclass)
# Collect results
preliminairy_results = \
wait_and_collect_results(client, task_preliminairy_results.get('id'))
crude_rate = [res["crude_rate"] for res in preliminairy_results]
total_local_pop = \
[res["total_local_population"] for res in preliminairy_results]
total_local_incidence = \
[res["total_local_incidence"] for res in preliminairy_results]
incidence_population_results = \
[res["incidence_population"] for res in preliminairy_results]
info("Combining crude rates")
crude_rate_combined = \
combined_crude_rate(total_local_incidence, total_local_pop)
info("Calculating people at risk")
people_at_risk_results = people_at_risk(incidence_population_results)
info("Dispatching Adjusted Rate")
task_adjusted_rate = client.create_new_task(
input_={
"method": "adjusted_rate",
"kwargs": {
"incidence": incidence,
"gender": gender,
"ageclass": ageclass,
"rel_pop": rel_pop,
"people_at_risk": people_at_risk_results,
"population": population,
"prefacture": prefacture
}
},
organization_ids=organization_ids
)
adjusted_rate_results = \
wait_and_collect_results(client, task_adjusted_rate.get('id'))
adjusted_rate_glob = \
[res["adj_rate_glob"] for res in adjusted_rate_results]
adjusted_rate_local = \
[res["adj_rate_local"] for res in adjusted_rate_results]
info("Combining adjusted rates")
adjusted_rate_combined = combined_adjusted_rate(adjusted_rate_glob)
info("Master algorithm complete")
return {
"local_crude_rate": crude_rate,
"combined_crude_rate": crude_rate_combined,
"local_adjusted_rate": adjusted_rate_local,
"combined_adjusted_rate": adjusted_rate_combined
}
def cli_server_files(ctx):
"""List files locations of a server instance."""
info(f"Configuration file = {ctx.config_file}")
info(f"Log file = {ctx.log_file}")
info(f"Database = {ctx.get_database_uri()}")
