def cli_node_stop(name, system_folders, all_nodes):
"""Stop a running container. """
client = docker.from_env()
check_if_docker_deamon_is_running(client)
running_nodes = client.containers.list(
filters={"label": f"{APPNAME}-type=node"})
if not running_nodes:
warning("No nodes are currently running.")
return
running_node_names = [node.name for node in running_nodes]
if all_nodes:
for name in running_node_names:
container = client.containers.get(name)
container.kill()
info(f"Stopped the {Fore.GREEN}{name}{Style.RESET_ALL} Node.")
else:
if not name:
name = q.select("Select the node you wish to stop:",
choices=running_node_names).ask()
else:
post_fix = "system" if system_folders else "user"
name = f"{APPNAME}-{name}-{post_fix}"
if name in running_node_names:
container = client.containers.get(name)
container.kill()
info(f"Stopped the {Fore.GREEN}{name}{Style.RESET_ALL} Node.")
else:
error(f"{Fore.RED}{name}{Style.RESET_ALL} is not running?")
def cli_node_list():
"""Lists all nodes in the default configuration directory."""
client = docker.from_env()
check_if_docker_deamon_is_running(client)
running_nodes = client.containers.list(
filters={"label": f"{APPNAME}-type=node"})
running_node_names = []
for node in running_nodes:
running_node_names.append(node.name)
header = \
"\nName"+(21*" ") + \
"Environments"+(20*" ") + \
"Status"+(10*" ") + \
"System/User"
click.echo(header)
click.echo("-" * len(header))
running = Fore.GREEN + "Online" + Style.RESET_ALL
stopped = Fore.RED + "Offline" + Style.RESET_ALL
# system folders
configs, f1 = NodeContext.available_configurations(system_folders=True)
for config in configs:
status = running if f"{APPNAME}-{config.name}-system" in \
running_node_names else stopped
click.echo(f"{config.name:25}"
f"{str(config.available_environments):32}"
f"{status:25} System ")
# user folders
configs, f2 = NodeContext.available_configurations(system_folders=False)
for config in configs:
status = running if f"{APPNAME}-{config.name}-user" in \
running_node_names else stopped
click.echo(f"{config.name:25}"
f"{str(config.available_environments):32}"
f"{status:25} User ")
click.echo("-" * 85)
if len(f1) + len(f2):
warning(
f"{Fore.RED}Failed imports: {len(f1)+len(f2)}{Style.RESET_ALL}")
def cli_node_list():
"""Lists all nodes in the default configuration directories."""
# FIXME: use package 'table' for this.
click.echo("\nName" + (21 * " ") + "Environments" + (21 * " ") +
"System/User")
click.echo("-" * 70)
configs, f1 = NodeContext.available_configurations(system_folders=True)
for config in configs:
click.echo(f"{config.name:25}{str(config.available_environments):32} "
f"System ")
configs, f2 = NodeContext.available_configurations(system_folders=False)
for config in configs:
click.echo(f"{config.name:25}{str(config.available_environments):32} "
f"User ")
click.echo("-" * 70)
warning(f"Number of failed imports: "
f"{Fore.YELLOW}{len(f1)+len(f2)}{Style.RESET_ALL}")
def cli_server_configuration_list():
"""Print the available configurations."""
click.echo("\nName"+(21*" ")+"Environments"+(21*" ")+"System/User")
click.echo("-"*70)
sys_configs, f1 = ServerContext.available_configurations(
system_folders=True)
for config in sys_configs:
click.echo(f"{config.name:25}{str(config.available_environments):32} System ")
usr_configs, f2 = ServerContext.available_configurations(system_folders=False)
for config in usr_configs:
click.echo(f"{config.name:25}{str(config.available_environments):32} User ")
click.echo("-"*70)
if len(f1)+len(f2):
warning(
f"{Fore.YELLOW}Number of failed imports: "
f"{len(f1)+len(f2)}{Style.RESET_ALL}"
)
def cli_node_create_private_key(name, environment, system_folders, upload,
organization_name, overwrite):
"""Create and upload a new private key (use with caughtion)"""
# retrieve context
name, environment = (name, environment) if name else \
select_configuration_questionaire("node", system_folders)
# raise error if config could not be found
NodeContext.LOGGING_ENABLED = False
if not NodeContext.config_exists(name, environment, system_folders):
error(f"The configuration {Fore.RED}{name}{Style.RESET_ALL} with "
f"environment {Fore.RED}{environment}{Style.RESET_ALL} could "
f"not be found.")
exit(1)
# Create node context
ctx = NodeContext(name, environment, system_folders)
# Authenticate with the server to obtain organization name if it wasn't
# provided
if organization_name is None:
client = create_client_and_authenticate(ctx)
organization_name = client.whoami.organization_name
# generate new key, and save it
filename = f"privkey_{organization_name}.pem"
file_ = ctx.type_data_folder(system_folders) / filename
if file_.exists():
warning(f"File '{Fore.CYAN}{file_}{Style.RESET_ALL}' exists!")
if overwrite:
warning("'--override' specified, so it will be overwritten ...")
if file_.exists() and not overwrite:
error("Could not create private key!")
warning(f"If you're **sure** you want to create a new key, "
f"please run this command with the '--overwrite' flag")
warning("Continuing with existing key instead!")
private_key = RSACryptor(file_).private_key
else:
try:
info("Generating new private key")
private_key = RSACryptor.create_new_rsa_key(file_)
except Exception as e:
error(f"Could not create new private key '{file_}'!?")
debug(e)
info("Bailing out ...")
exit(1)
warning(f"Private key written to '{file_}'")
warning(
"If you're running multiple nodes, be sure to copy the private "
"key to the appropriate directories!")
# create public key
info("Deriving public key")
public_key = RSACryptor.create_public_key_bytes(private_key)
# update config file
info("Updating configuration")
ctx.config["encryption"]["private_key"] = str(file_)
ctx.config_manager.put(environment, ctx.config)
ctx.config_manager.save(ctx.config_file)
# upload key to the server
if upload:
info("Uploading public key to the server. "
"This will overwrite any previously existing key!")
if 'client' not in locals():
client = create_client_and_authenticate(ctx)
try:
client.request(f"/organization/{client.whoami.organization_id}",
method="patch",
json={"public_key": bytes_to_base64s(public_key)})
except Exception as e:
error("Could not upload the public key!")
debug(e)
exit(1)
else:
warning("Public key not uploaded!")
info("[Done]")
def cli_node_start(name, config, environment, system_folders, image, keep,
mount_src):
"""Start the node instance.
If no name or config is specified the default.yaml configuation is
used. In case the configuration file not excists, a questionaire is
invoked to create one. Note that in this case it is not possible to
specify specific environments for the configuration (e.g. test,
prod, acc).
"""
info("Starting node...")
info("Finding Docker deamon")
docker_client = docker.from_env()
check_if_docker_deamon_is_running(docker_client)
if config:
name = Path(config).stem
ctx = NodeContext(name, environment, system_folders, config)
else:
# in case no name is supplied, ask the user to select one
if not name:
name, environment = select_configuration_questionaire(
"node", system_folders)
# check that config exists, if not a questionaire will be invoked
if not NodeContext.config_exists(name, environment, system_folders):
question = f"Configuration '{name}' using environment"
question += f" '{environment}' does not exist.\n Do you want to"
question += f" create this config now?"
if q.confirm(question).ask():
configuration_wizard("node", name, environment, system_folders)
else:
error("Config file couldn't be loaded")
sys.exit(0)
NodeContext.LOGGING_ENABLED = False
ctx = NodeContext(name, environment, system_folders)
# check that this node is not already running
running_nodes = docker_client.containers.list(
filters={"label": f"{APPNAME}-type=node"})
suffix = "system" if system_folders else "user"
for node in running_nodes:
if node.name == f"{APPNAME}-{name}-{suffix}":
error(f"Node {Fore.RED}{name}{Style.RESET_ALL} is already running")
exit(1)
# make sure the (host)-task and -log dir exists
info("Checking that data and log dirs exist")
ctx.data_dir.mkdir(parents=True, exist_ok=True)
ctx.log_dir.mkdir(parents=True, exist_ok=True)
if image is None:
image = ctx.config.get(
"image", "harbor.vantage6.ai/infrastructure/node:latest")
info(f"Pulling latest node image '{image}'")
try:
# docker_client.images.pull(image)
pull_if_newer(image)
except Exception:
warning(' ... alas, no dice!')
else:
info(" ... success!")
info("Creating Docker data volume")
data_volume = docker_client.volumes.create(
f"{ctx.docker_container_name}-vol")
info("Creating file & folder mounts")
# FIXME: should only mount /mnt/database.csv if it is a file!
# FIXME: should obtain mount points from DockerNodeContext
mounts = [
# (target, source)
("/mnt/database.csv", str(ctx.databases["default"])),
("/mnt/log", str(ctx.log_dir)),
("/mnt/data", data_volume.name),
("/mnt/config", str(ctx.config_dir)),
("/var/run/docker.sock", "/var/run/docker.sock"),
]
if mount_src:
# If mount_src is a relative path, docker willl consider it a volume.
mount_src = os.path.abspath(mount_src)
mounts.append(('/vantage6/vantage6-node', mount_src))
# FIXME: Code duplication: Node.__init__() (vantage6/node/__init__.py)
# uses a lot of the same logic. Suggest moving this to
# ctx.get_private_key()
filename = ctx.config.get("encryption", {}).get("private_key")
# filename may be set to an empty string
if not filename:
filename = 'private_key.pem'
# Location may be overridden by the environment
filename = os.environ.get('PRIVATE_KEY', filename)
# If ctx.get_data_file() receives an absolute path, it is returned as-is
fullpath = Path(ctx.get_data_file(filename))
if fullpath:
if Path(fullpath).exists():
mounts.append(("/mnt/private_key.pem", str(fullpath)))
else:
warning(f"private key file provided {fullpath}, "
"but does not exists")
volumes = {}
for mount in mounts:
volumes[mount[1]] = {'bind': mount[0], 'mode': 'rw'}
# Be careful not to use 'environment' as it would override the function
# argument ;-).
env = {
"DATA_VOLUME_NAME": data_volume.name,
"DATABASE_URI": "/mnt/database.csv",
"PRIVATE_KEY": "/mnt/private_key.pem"
}
cmd = f'vnode-local start -c /mnt/config/{name}.yaml -n {name} -e '\
f'{environment} --dockerized'
info(f"Runing Docker container")
# debug(f" with command: '{cmd}'")
# debug(f" with mounts: {volumes}")
# debug(f" with environment: {env}")
container = docker_client.containers.run(image,
command=cmd,
volumes=volumes,
detach=True,
labels={
f"{APPNAME}-type": "node",
"system": str(system_folders),
"name": ctx.config_file_name
},
environment=env,
name=ctx.docker_container_name,
auto_remove=not keep,
tty=True)
info(f"Success! container id = {container}")
def RPC_adjusted_rate(data, gender, ageclass, incidence, rel_pop,
people_at_risk, population, prefacture):
"""Calculates the adjusted rate for the ASR.
This does this by first creating a pivot of the local node's data using
the incidence by age class and gender. Then it is multiplied by rel_pop,
a dataframe created using the function Relative_population. This is all
divided by the people at risk which is the total population at risk and
summed across the columns.
Parameters
----------
data : pd.DataFrame
Dataframe containing incidence data per ageclass and gender
gender : pd.DataFrame column
The different genders
ageclass : pd.DataFrame column
The different age classes
incidence : pd.DataFrame column
Column containing the incidence (deaths)
rel_pop : pd.DataFrame
Standardised population of 100000
people_at_risk : pd.DataFrame
Combined dataframe of all the people at risk of catching the disease
across all nodes
population: pd.DataFrame column
Column containing the population per age category and gender at the
node
Returns
-------
dictionary
Returns a dictionary object of pd.core.Series containing the adjusted
rate based on local incidence population and adjusted rate based on
global population. Depending on how many genders are listed, these will
be of length len(np.unique(data[gender])).
"""
# dislcosive check
n = len(data)
if n < 10:
warning('Dataset is too small: len(data) < 10. Exiting.')
return
incidence_per_ageclass_and_gender = data.pivot(
index=[prefacture, gender],
columns=ageclass,
values=incidence
)
adj_rate_glob = incidence_per_ageclass_and_gender.mul(rel_pop).\
div(people_at_risk).sum(axis=1)
incidence_pop = data.pivot(index=gender, columns=ageclass,
values=population)
adj_rate_local = incidence_per_ageclass_and_gender.mul(rel_pop).\
div(incidence_pop).sum(axis=1)
return {
"adj_rate_local": adj_rate_local,
"adj_rate_glob": adj_rate_glob
}
def RPC_preliminairy_results(data, incidence, population, gender, ageclass,
prefacture=None):
"""Calculates the crude rate as well as preliminary statistics used for
calculating the combined crude rate.
Parameters
----------
data : pd.DataFrame
Pandas Dataframe containing incidencer per age class and gender
incidence : pd.DataFrame column
Column containing the incidence (deaths) at each node
population : pd.DataFrame column
Column containing the population at each node
gender : pd.DataFrame column
Column containing the different genders at each node
ageclass : pd.DataFrame column
Column containing the ageclass's at each node
prefacture : pd.DataFrame column, optional
Column containing the prefacture, by default None
Returns
-------
Dictionairy
Returns a dictionairy object containing the crude rate at the node,
the total number of incidence cases 'incidence pop' by age class and
gender, summed population 'total_local_pop' at the node (this is
indifferent of ageclass and gender) as well as the summed incidence
(this is also indifferent of ageclass and gender)
"""
# disclosive check
n = len(data)
if n < 10:
warning('Dataset is too small: len(data) < 10. Exiting')
return
if prefacture:
people_at_risk = data.pivot(
index=[prefacture, gender],
columns=ageclass,
values=population
).sum(axis=1)
incidence_per_prefacture = data.pivot(
index=[prefacture, gender],
columns=ageclass,
values=incidence
).sum(axis=1)
crd_rate = (incidence_per_prefacture.div(people_at_risk))*100000
else:
people_at_risk = data.pivot_table(
index=gender,
columns=ageclass,
values=population
).sum(axis=1)
incidence_per_gender_age = data.pivot_table(
index=gender,
columns=ageclass,
values=incidence
).sum(axis=1)
crd_rate = (incidence_per_gender_age.div(people_at_risk))*100000
incidence_pop = data.pivot(index=gender, columns=ageclass,
values=population)
total_local_pop = incidence_pop.sum(axis=1)
total_local_inc = data.pivot(index=gender, columns=ageclass,
values=incidence).sum(axis=1)
return {
"crude_rate": crd_rate,
"incidence_population": incidence_pop,
"total_local_population": total_local_pop,
"total_local_incidence": total_local_inc
}
def cli_server_import(ctx, file_, drop_all, image, keep):
""" Import organizations/collaborations/users and tasks.
Especially usefull for testing purposes.
"""
info("Starting server...")
info("Finding Docker daemon.")
docker_client = docker.from_env()
# will print an error if not
check_if_docker_deamon_is_running(docker_client)
# pull lastest Docker image
if image is None:
image = ctx.config.get(
"image", "harbor.vantage6.ai/infrastructure/server:latest")
info(f"Pulling latest server image '{image}'.")
try:
docker_client.images.pull(image)
except Exception:
warning("... alas, no dice!")
else:
info(" ... succes!")
info("Creating mounts")
mounts = [
docker.types.Mount("/mnt/config.yaml",
str(ctx.config_file),
type="bind"),
docker.types.Mount("/mnt/import.yaml", str(file_), type="bind")
]
# FIXME: code duplication with cli_server_start()
# try to mount database
uri = ctx.config['uri']
url = make_url(uri)
environment_vars = None
# If host is None, we're dealing with a file-based DB, like SQLite
if (url.host is None):
db_path = url.database
if not os.path.isabs(db_path):
# We're dealing with a relative path here -> make it absolute
db_path = ctx.data_dir / url.database
basename = os.path.basename(db_path)
dirname = os.path.dirname(db_path)
os.makedirs(dirname, exist_ok=True)
# we're mounting the entire folder that contains the database
mounts.append(
docker.types.Mount("/mnt/database/", dirname, type="bind"))
environment_vars = {
"VANTAGE6_DB_URI": f"sqlite:////mnt/database/{basename}"
}
else:
warning(f"Database could not be transfered, make sure {url.host} "
"is reachable from the Docker container")
info("Consider using the docker-compose method to start a server")
drop_all_ = "--drop-all" if drop_all else ""
cmd = f'vserver-local import -c /mnt/config.yaml -e {ctx.environment} ' \
f'{drop_all_} /mnt/import.yaml'
info(cmd)
info("Run Docker container")
container = docker_client.containers.run(image,
command=cmd,
mounts=mounts,
detach=True,
labels={
f"{APPNAME}-type": "server",
"name": ctx.config_file_name
},
environment=environment_vars,
auto_remove=not keep,
tty=True)
logs = container.logs(stream=True, stdout=True)
Thread(target=print_log_worker, args=(logs, ), daemon=False).start()
info(f"Succes! container id = {container.id}")
def cli_server_start(ctx, ip, port, debug, image, keep):
"""Start the server."""
info("Starting server...")
info("Finding Docker daemon.")
docker_client = docker.from_env()
# will print an error if not
check_if_docker_deamon_is_running(docker_client)
# check that this server is not already running
running_servers = docker_client.containers.list(
filters={"label": f"{APPNAME}-type=server"})
for server in running_servers:
if server.name == f"{APPNAME}-{ctx.name}-{ctx.scope}-server":
error(f"Server {Fore.RED}{ctx.name}{Style.RESET_ALL} "
"is already running")
exit(1)
# pull the server docker image
if image is None:
image = ctx.config.get(
"image", "harbor.vantage6.ai/infrastructure/server:latest")
info(f"Pulling latest server image '{image}'.")
try:
pull_if_newer(image)
# docker_client.images.pull(image)
except Exception:
warning("... alas, no dice!")
else:
info(" ... succes!")
info("Creating mounts")
mounts = [
docker.types.Mount("/mnt/config.yaml",
str(ctx.config_file),
type="bind")
]
# FIXME: code duplication with cli_server_import()
# try to mount database
uri = ctx.config['uri']
url = make_url(uri)
environment_vars = None
# If host is None, we're dealing with a file-based DB, like SQLite
if (url.host is None):
db_path = url.database
if not os.path.isabs(db_path):
# We're dealing with a relative path here -> make it absolute
db_path = ctx.data_dir / url.database
basename = os.path.basename(db_path)
dirname = os.path.dirname(db_path)
os.makedirs(dirname, exist_ok=True)
# we're mounting the entire folder that contains the database
mounts.append(
docker.types.Mount("/mnt/database/", dirname, type="bind"))
environment_vars = {
"VANTAGE6_DB_URI": f"sqlite:////mnt/database/{basename}"
}
else:
warning(f"Database could not be transfered, make sure {url.host} "
"is reachable from the Docker container")
info("Consider using the docker-compose method to start a server")
ip_ = f"--ip {ip}" if ip else ""
port_ = f"--port {port}" if port else ""
cmd = f'vserver-local start -c /mnt/config.yaml -e {ctx.environment} ' \
f'{ip_} {port_}'
info(cmd)
info("Run Docker container")
port_ = str(port or ctx.config["port"] or 5000)
container = docker_client.containers.run(
image,
command=cmd,
mounts=mounts,
detach=True,
labels={
f"{APPNAME}-type": "server",
"name": ctx.config_file_name
},
environment=environment_vars,
ports={f"{port_}/tcp": ("127.0.0.1", port_)},
name=ctx.docker_container_name,
auto_remove=not keep,
tty=True)
info(f"Succes! container id = {container}")