def get_hosts():
config = App.load_config()
return [config.get('esearch', {})]
from tornado.httpserver import HTTPServer
from tornado import ioloop, netutil, process
from tornado.options import define, options as opt
import controller as c
from controller.app import Application
from utils.helper import prop
define('num_processes', default=4, help='sub-processes count', type=int)
if __name__ == '__main__':
os.chdir(os.path.dirname(os.path.realpath(__file__)))
opt.parse_command_line()
routes = c.handlers + c.views
app = Application(routes,
default_handler_class=c.InvalidPageHandler,
ui_modules=c.modules,
xsrf_cookies=True)
try:
ssl_options = not opt.debug and app.site.get('https') or None
server = HTTPServer(app, xheaders=True, ssl_options=ssl_options)
sockets = netutil.bind_sockets(opt.port, family=socket.AF_INET)
fork_id = 0 if opt.debug or os.name == 'nt' else process.fork_processes(
opt.num_processes)
server.add_sockets(sockets)
protocol = 'https' if ssl_options else 'http'
logging.info('Start the service #%d v%s on %s://localhost:%d' %
(fork_id, app.version, protocol, opt.port))
if fork_id == 0 and app.db and app.db.page.count_documents(
{}) == 0 and os.path.exists(app.config.get('data_path', '')):
script = 'python3 utils/add_pages.py --uri={0} --db_name={1} --json_path={2}/json'.format(
app.db_uri, app.config['database']['name'],
def reload(
services: List[str] = typer.Argument(
None,
help="Services to be reloaded",
shell_complete=Application.autocomplete_service,
),
) -> None:
Application.print_command(Application.serialize_parameter("", services))
Application.get_controller().controller_init(services)
docker = Docker()
running_services = docker.get_running_services()
if "frontend" in services and len(services) > 1:
print_and_exit("Can't reload frontend and other services at once")
reloaded = 0
for service in Application.data.services:
# Special case: frontend in production mode
if Configuration.production and service == "frontend":
# Only consider it if explicitly requested in input
if "frontend" not in services:
log.debug(
"Can't reload the frontend if not explicitly requested")
else:
log.info("Reloading frontend...")
# The frontend build stucks in swarm mode... let's start the container
# always in compose mode when using the reload comand
Configuration.FORCE_COMPOSE_ENGINE = True
Application.get_controller().controller_init([service])
docker = Docker()
docker.compose.start_containers([service], force=True)
reloaded += 1
continue
if service not in running_services:
continue
containers = docker.get_containers(service)
if not containers:
log.warning("Can't find any container for {}", service)
continue
try:
# get the first container from the containers dict
container = containers.get(list(containers.keys())[0])
# Just added for typing purpose
if not container: # pragma: no conver
log.warning("Can't find any container for {}", service)
continue
output = docker.exec_command(
container,
user="******",
command="ls /usr/local/bin/reload",
force_output_return=True,
)
# this is to consume the iterator and raise the exception with exit code
if output:
[_ for _ in output]
except DockerException as e:
# fail2ban fails with code 1
if "It returned with code 1" in str(e):
log.warning("Service {} does not support the reload command",
service)
continue
# backend fails with code 2
if "It returned with code 2" in str(e):
log.warning("Service {} does not support the reload command",
service)
continue
raise
docker.exec_command(containers,
user="******",
command="/usr/local/bin/reload")
reloaded += 1
if reloaded == 0:
log.info("No service reloaded")
else:
log.info("Services reloaded")
def init(
create_projectrc: bool = typer.Option(
False,
"--force",
"-f",
help="Overwrite initialization files if already exist",
show_default=False,
),
submodules_path: Path = typer.Option(
None,
"--submodules-path",
help=
"Link all submodules in an existing folder instead of download them",
),
) -> None:
Application.print_command(
Application.serialize_parameter("--force",
create_projectrc,
IF=create_projectrc),
Application.serialize_parameter("--submodules-path",
submodules_path,
IF=submodules_path),
)
Application.get_controller().controller_init()
for p in Application.project_scaffold.data_folders:
if not p.exists():
p.mkdir(parents=True, exist_ok=True)
for p in Application.project_scaffold.data_files:
if not p.exists():
p.touch()
if not Configuration.projectrc and not Configuration.host_configuration:
create_projectrc = True
# We have to create the .projectrc twice
# One generic here with main options and another after the complete
# conf reading to set services variables
if create_projectrc:
Application.get_controller().create_projectrc()
Application.get_controller().read_specs(read_extended=False)
if submodules_path is not None:
if not submodules_path.exists():
print_and_exit("Local path not found: {}", submodules_path)
Application.git_submodules(from_path=submodules_path)
Application.get_controller().read_specs(read_extended=True)
Application.get_controller().make_env()
# Compose services and variables
Application.get_controller().get_compose_configuration()
# We have to create the .projectrc twice
# One generic with main options and another here
# when services are available to set specific configurations
if create_projectrc:
Application.get_controller().create_projectrc()
Application.get_controller().read_specs(read_extended=True)
Application.get_controller().make_env()
if Configuration.swarm_mode:
docker = Docker(verify_swarm=False)
if not docker.swarm.get_token():
docker.swarm.init()
log.info("Swarm is now initialized")
else:
log.debug("Swarm is already initialized")
if Configuration.frontend == ANGULAR:
yarn_lock = DATA_DIR.joinpath(Configuration.project, "frontend",
"yarn.lock")
if yarn_lock.exists():
yarn_lock.unlink()
log.info("Yarn lock file deleted")
log.info("Project initialized")
def test_autocomplete(capfd: Capture, faker: Faker) -> None:
create_project(
capfd=capfd,
name=random_project_name(faker),
)
init_project(capfd)
app = Application()
values = app.autocomplete_service(None, None, "") # type: ignore
assert len(values) > 0
assert "backend" in values
values = app.autocomplete_service(None, None, "invalid") # type: ignore
assert len(values) == 0
values = app.autocomplete_service(None, None, "b") # type: ignore
assert len(values) >= 1
assert "backend" in values
values = app.autocomplete_allservice(None, None, "") # type: ignore
assert len(values) > 0
assert "backend" in values
values = app.autocomplete_allservice(None, None, "invalid") # type: ignore
assert len(values) == 0
values = app.autocomplete_allservice(None, None, "b") # type: ignore
assert len(values) >= 1
assert "backend" in values
values = app.autocomplete_allservice(None, None, "c") # type: ignore
assert len(values) >= 1
assert "backend" not in values
values = app.autocomplete_submodule(None, None, "") # type: ignore
assert len(values) > 0
assert "main" in values
values = app.autocomplete_submodule(None, None, "invalid") # type: ignore
assert len(values) == 0
values = app.autocomplete_submodule(None, None, "m") # type: ignore
assert len(values) >= 1
assert "main" in values
values = app.autocomplete_submodule(None, None, "d") # type: ignore
assert len(values) >= 1
assert "main" not in values
os.unlink(".rapydo")
values = app.autocomplete_service(None, None, "") # type: ignore
assert len(values) == 0
values = app.autocomplete_allservice(None, None, "") # type: ignore
assert len(values) == 0
values = app.autocomplete_submodule(None, None, "") # type: ignore
assert len(values) == 0
def create_volatile_container(
self,
service: str,
command: Optional[str] = None,
publish: Optional[List[Union[PortMapping, PortRangeMapping]]] = None,
# used by interfaces
detach: bool = False,
user: Optional[str] = None,
) -> bool:
compose_engine_forced = False
if Configuration.swarm_mode:
# import here to prevent circular imports
from controller.app import Application
if not Configuration.FORCE_COMPOSE_ENGINE:
compose_engine_forced = True
Configuration.FORCE_COMPOSE_ENGINE = True
# init is needed to reload the configuration to force compose engine
Application.get_controller().controller_init()
tty = sys.stdout.isatty()
try:
output = self.docker.compose.run(
service=service,
name=service,
command=Docker.split_command(command),
user=user,
detach=detach,
tty=tty and not detach,
stream=not tty and not detach,
dependencies=False,
remove=True,
service_ports=False,
publish=publish or [],
use_aliases=True,
)
if not detach:
for out_line in output: # type: ignore
# 'stdout' or 'stderr'
# Both out and err are collapsed in stdout
# Maybe in the future would be useful to keep them separated?
# stdstream = out_line[0]
line = out_line[1]
if isinstance(line, bytes):
line = line.decode("UTF-8")
print(line.strip())
if compose_engine_forced:
Configuration.FORCE_COMPOSE_ENGINE = False
# init is needed to reload the configuration to undo compose engine
Application.get_controller().controller_init()
return True
except DockerException as e:
log.critical(e)
return False
def get_app(self, testing=True, debug=False, extra_handlers=None):
options.testing = testing
options.debug = debug
options.port = self.get_http_port()
return Application(c.handlers + c.views + (extra_handlers or []), db_name_ext='_test',
ui_modules=c.modules, default_handler_class=c.InvalidPageHandler)
def shell(
service: str = typer.Argument(
...,
help="Service name",
shell_complete=Application.autocomplete_service),
command: str = typer.Argument(
"bash",
help="UNIX command to be executed on selected running service"),
user: Optional[str] = typer.Option(
None,
"--user",
"-u",
help="User existing in selected service",
show_default=False,
),
default_command: bool = typer.Option(
False,
"--default-command",
"--default",
help="Execute the default command configured for the container",
show_default=False,
),
no_tty: bool = typer.Option(
False,
"--no-tty",
help=
"Disable pseudo-tty allocation (useful for non-interactive script)",
show_default=False,
),
replica: int = typer.Option(
1,
"--replica",
"--slot",
help="Execute the command on the specified replica",
show_default=False,
),
broadcast: bool = typer.Option(
False,
"--broadcast",
help="Execute the command on all the replicas",
show_default=False,
),
) -> None:
Application.print_command(
Application.serialize_parameter("--user", user, IF=user),
Application.serialize_parameter("--default",
default_command,
IF=default_command),
Application.serialize_parameter("", service),
Application.serialize_parameter("", command),
)
if no_tty:
log.warning("--no-tty option is deprecated, you can stop using it")
if replica > 1 and broadcast:
print_and_exit("--replica and --broadcast options are not compatible")
Application.get_controller().controller_init()
docker = Docker()
if not user:
user = services.get_default_user(service)
if default_command:
command = services.get_default_command(service)
log.debug("Requested command: {} with user: {}", command, user
or "default")
if broadcast:
containers = docker.get_containers(service)
if not containers:
print_and_exit("No running container found for {} service",
service)
docker.exec_command(containers, user=user, command=command)
else:
container = docker.get_container(service, slot=replica)
if not container:
if replica != 1:
print_and_exit("Replica number {} not found for {} service",
str(replica), service)
print_and_exit("No running container found for {} service",
service)
docker.exec_command(container, user=user, command=command)
def remove(
services: List[str] = typer.Argument(
None,
help="Services to be removed",
shell_complete=Application.autocomplete_service,
),
rm_all: bool = typer.Option(
False,
"--all",
help="Also remove persistent data stored in docker volumes",
show_default=False,
),
) -> None:
Application.print_command(
Application.serialize_parameter("--all", rm_all, IF=rm_all),
Application.serialize_parameter("", services),
)
remove_extras: List[str] = []
for extra in (
REGISTRY,
"adminer",
"swaggerui",
):
if services and extra in services:
# services is a tuple, even if defined as List[str] ...
services = list(services)
services.pop(services.index(extra))
remove_extras.append(extra)
Application.get_controller().controller_init(services)
docker = Docker()
if remove_extras:
for extra_service in remove_extras:
if not docker.client.container.exists(extra_service):
log.error("Service {} is not running", extra_service)
continue
docker.client.container.remove(extra_service, force=True)
log.info("Service {} removed", extra_service)
# Nothing more to do
if not services:
return
all_services = Application.data.services == Application.data.active_services
if all_services and rm_all:
# Networks are not removed, but based on docker compose down --help they should
# Also docker-compose down removes network from what I remember
# Should be reported as bug? If corrected a specific check in test_remove.py
# will start to fail
docker.client.compose.down(
remove_orphans=False,
remove_images="local",
# Remove named volumes declared in the volumes section of the
# Compose file and anonymous volumes attached to containers.
volumes=rm_all,
)
else:
# Important note: volumes=True only destroy anonymous volumes,
# not named volumes like down should do
docker.client.compose.rm(Application.data.services,
stop=True,
volumes=rm_all)
log.info("Stack removed")
def password(
service: SupportedServices = typer.Argument(None, help="Service name"),
show: bool = typer.Option(
False,
"--show",
help="Show the current password(s)",
show_default=False,
),
random: bool = typer.Option(
False,
"--random",
help="Generate a random password",
show_default=False,
),
new_password: str = typer.Option(
None,
"--password",
help="Force the given password",
show_default=False,
),
) -> None:
Application.print_command(
Application.serialize_parameter("--show", show, IF=show),
Application.serialize_parameter("--random", random, IF=random),
Application.serialize_parameter("--password",
new_password,
IF=new_password),
Application.serialize_parameter("", service),
)
Application.get_controller().controller_init()
# No service specified, only a summary will be reported
if not service:
if random:
print_and_exit("--random flag is not supported without a service")
if new_password:
print_and_exit(
"--password option is not supported without a service")
MIN_PASSWORD_SCORE = int(
Application.env.get("MIN_PASSWORD_SCORE", 2) # type: ignore
)
last_updates = parse_projectrc()
now = datetime.now()
table: List[List[str]] = []
for s in PASSWORD_MODULES:
# This should never happens and can't be (easily) tested
if s not in Application.data.base_services: # pragma: no cover
print_and_exit("Command misconfiguration, unknown {} service",
s)
if s != REGISTRY and s not in Application.data.active_services:
continue
if s == REGISTRY and not Configuration.swarm_mode:
continue
module = PASSWORD_MODULES.get(s)
if not module: # pragma: no cover
print_and_exit(f"{s} misconfiguration, module not found")
for variable in module.PASSWORD_VARIABLES:
password = Application.env.get(variable)
if password == PLACEHOLDER:
score = None
else:
result = zxcvbn(password)
score = result["score"]
if variable in last_updates:
change_date = last_updates.get(variable,
datetime.fromtimestamp(0))
expiration_date = change_date + timedelta(
days=PASSWORD_EXPIRATION)
expired = now > expiration_date
last_change = change_date.strftime("%Y-%m-%d")
else:
expired = True
last_change = "N/A"
pass_line: List[str] = []
pass_line.append(s)
pass_line.append(variable)
if expired:
pass_line.append(RED(last_change))
else:
pass_line.append(GREEN(last_change))
if score is None:
pass_line.append(RED("NOT SET"))
elif score < MIN_PASSWORD_SCORE:
pass_line.append(RED(score))
else:
pass_line.append(GREEN(score))
if show:
pass_line.append(str(password))
table.append(pass_line)
headers = ["SERVICE", "VARIABLE", "LAST CHANGE", "STRENGTH"]
if show:
headers.append("PASSWORD")
print("")
print(tabulate(
table,
tablefmt=TABLE_FORMAT,
headers=headers,
))
# In this case a service is asked to be updated
else:
module = PASSWORD_MODULES.get(service.value)
if not module: # pragma: no cover
print_and_exit(
f"{service.value} misconfiguration, module not found")
if random:
new_password = get_strong_password()
elif not new_password:
print_and_exit(
"Please specify one between --random and --password options")
docker = Docker()
variables = module.PASSWORD_VARIABLES
old_password = Application.env.get(variables[0])
new_variables = {variable: new_password for variable in variables}
# Some services can only be updated if already running,
# others can be updated even if offline,
# but in every case if the stack is running it has to be restarted
if service.value == REGISTRY:
is_running = docker.registry.ping(do_exit=False)
container: Optional[Tuple[str, str]] = ("registry", "")
else:
container = docker.get_container(service.value)
is_running = container is not None
is_running_needed = module.IS_RUNNING_NEEDED
log.info("Changing password for {}...", service.value)
if is_running_needed and (not is_running or not container):
print_and_exit(
"Can't update {} because it is not running. Please start your stack",
service.value,
)
update_projectrc(new_variables)
if container:
module.password(container, old_password, new_password)
if is_running:
log.info("{} was running, restarting services...", service.value)
Application.get_controller().check_placeholders_and_passwords(
Application.data.compose_config, Application.data.services)
if service.value == REGISTRY:
port = cast(int, Application.env["REGISTRY_PORT"])
docker.client.container.remove(REGISTRY, force=True)
docker.compose.create_volatile_container(REGISTRY,
detach=True,
publish=[(port, port)
])
elif Configuration.swarm_mode:
docker.compose.dump_config(Application.data.services)
docker.swarm.deploy()
else:
docker.compose.start_containers(Application.data.services)
else:
log.info("{} was not running, restart is not needed",
service.value)
log.info(
"The password of {} has been changed. "
"Please find the new password into your .projectrc file as {} variable",
service.value,
variables[0],
)
