async def build_image(app_name, version):
tag_name = f"{static.APP_PREFIX}_{app_name}"
repo = f"{config.DOCKER_REGISTRY}/{tag_name}:{version}"
try:
pathlib.Path(f"./temp_apps/{app_name}/{version}/src").mkdir(parents=True, exist_ok=True)
except Exception as e:
print(e)
minio_client = Minio(config.MINIO, access_key=config.get_from_file(config.MINIO_ACCESS_KEY_PATH),
secret_key=config.get_from_file(config.MINIO_SECRET_KEY_PATH), secure=False)
objects = minio_client.list_objects("apps-bucket", recursive=True)
for obj in objects:
size = obj.size
p_src = Path(obj.object_name)
if p_src.parts[1] == app_name:
hold = str(p_src)
p_dst = hold[hold.find(app_name):]
p_dst = Path("temp_apps") / p_dst
os.makedirs(p_dst.parent, exist_ok=True)
data = minio_client.get_object('apps-bucket', hold)
with open(p_dst, 'wb+') as file_data:
for d in data.stream(size):
file_data.write(d)
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
async with connect_to_aiodocker() as docker_client:
context_dir = f"./temp_apps/{app_name}/{version}/"
with docker_context(Path(context_dir)) as context:
logger.info("Sending image to be built")
dockerfile = "./Dockerfile"
try:
log_stream = await docker_client.images.build(fileobj=context, tag=repo, rm=True,
forcerm=True, pull=True, stream=True,
path_dockerfile=dockerfile,
encoding="application/x-tar")
logger.info("Docker image building")
await stream_docker_log(log_stream)
logger.info("Docker image Built")
# if await push_image(docker_client, repo):
# return "Docker image built and pushed successfully."
success = await push_image(docker_client, repo)
if success:
return True, "Successfully built and pushed image"
else:
return False, "Failed to push image"
except Exception as e:
return False, str(e)
async def down(self):
# Set up a subcommand parser
parser = argparse.ArgumentParser(
description=
"Remove the WALKOFF stack and optionally related artifacts.")
parser.add_argument("-c",
"--clean",
action="store_true",
help="Removes all encryption keys, volumes, data.")
parser.add_argument("-d",
"--debug",
action="store_true",
help="Set log level to debug.")
parser.add_argument("-y",
"--yes",
action="store_true",
help="Skips all verification questions.")
# Parse out the command
args = parser.parse_args(sys.argv[2:])
if args.debug:
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
proc = await rm_stack("walkoff")
# if not args.skipnetwork:
# logger.info("Waiting for containers to exit and network to be removed...")
# await exponential_wait(check_for_network, [self.docker_client], "Network walkoff_default still exists")
if args.clean:
if args.yes or await are_you_sure(
"Are you sure you want to remove all WALKOFF data? This will remove "
"all of WALKOFF's docker secrets, docker volumes, and data for "
"walkoff_resource services,"):
for key in key_names:
await delete_encryption_key(self.docker_client, key)
for volume in volume_names:
await delete_volume(self.docker_client, volume)
logger.info(
"Walkoff stack removed, it may take a few seconds to stop all containers."
)
return proc.returncode
async def down(self):
# Set up a subcommand parser
parser = argparse.ArgumentParser(description="Remove the WALKOFF stack and optionally related artifacts.")
parser.add_argument("-k", "--key", action="store_true",
help="Removes the walkoff_encryption_key secret.")
parser.add_argument("-r", "--registry", action="store_true",
help="Clears the registry bind mount directory.")
parser.add_argument("-v", "--volume", action="store_true", help="Clears the postgresql volume")
parser.add_argument("-d", "--debug", action="store_true",
help="Set log level to debug.")
# Parse out the command
args = parser.parse_args(sys.argv[2:])
if args.debug:
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
proc = await rm_stack("walkoff")
# if not args.skipnetwork:
# logger.info("Waiting for containers to exit and network to be removed...")
# await exponential_wait(check_for_network, [self.docker_client], "Network walkoff_default still exists")
if args.key:
if await are_you_sure("Deleting encryption key will render database unreadable, so it will be cleared. "
"This will delete all workflows, execution results, globals, users, roles, etc. "):
await delete_encryption_key(self.docker_client, "walkoff_encryption_key")
await delete_encryption_key(self.docker_client, "walkoff_internal_key")
await delete_encryption_key(self.docker_client, "walkoff_postgres_key")
await delete_dir_contents(static.POSTGRES_DATA_PATH)
if args.registry:
await delete_dir_contents(static.REGISTRY_DATA_PATH)
await delete_dir_contents(static.MINIO_DATA_PATH)
await delete_encryption_key(self.docker_client, "walkoff_minio_access_key")
await delete_encryption_key(self.docker_client, "walkoff_minio_secret_key")
if args.volume:
await remove_volume("walkoff_postgres-data", wait=True)
logger.info("Walkoff stack removed, it may take a little time to stop all services. "
"It is OK if the walkoff_default network is not fully removed.")
return proc.returncode
async def build_image(app_name, version):
tag_name = f"{static.APP_PREFIX}_{app_name}"
repo = f"{config.DOCKER_REGISTRY}/{tag_name}:{version}"
try:
pathlib.Path(f"./temp_apps/{app_name}/{version}/src").mkdir(
parents=True, exist_ok=True)
except Exception as e:
print(e)
minio_client = Minio(config.MINIO,
access_key='walkoff',
secret_key='walkoff123',
secure=False)
objects = minio_client.list_objects("apps-bucket", recursive=True)
for obj in objects:
size = obj.size
p_src = Path(obj.object_name)
if p_src.parts[1] == app_name:
hold = str(p_src)
p_dst = hold[hold.find(app_name):]
p_dst = f"./temp_apps/{p_dst}"
data = minio_client.get_object('apps-bucket', hold)
with open(str(p_dst), 'wb') as file_data:
for d in data.stream(size):
file_data.write(d)
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
async with connect_to_aiodocker() as docker_client:
context_dir = f"./temp_apps/{app_name}/{version}/"
with docker_context(Path(context_dir)) as context:
logger.info("Sending image to be built")
dockerfile = "./Dockerfile"
log_stream = await docker_client.images.build(
fileobj=context,
tag=repo,
rm=True,
forcerm=True,
pull=True,
stream=True,
path_dockerfile=dockerfile,
encoding="application/x-tar")
logger.info("Docker image building")
await stream_docker_log(log_stream)
logger.info("Docker image Built")
await push_image(docker_client, repo)
async def run():
""" Landing pad to launch primary command and do whatever async init the bootloader needs. """
# TODO: fill in the helps, and further develop cli with the end user in mind
commands = {"up", "down", "refresh"}
parser = argparse.ArgumentParser()
parser.add_argument("command", choices=commands)
parser.add_argument("args", nargs=argparse.REMAINDER)
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
# Parse out the command
args = parser.parse_args(sys.argv[1:2])
async with aiohttp.ClientSession() as session, connect_to_aiodocker() as docker_client:
bootloader = Bootloader(session, docker_client)
if hasattr(bootloader, args.command):
await getattr(bootloader, args.command)()
else:
logger.error("Invalid command.")
# TODO: Pipe this through the logger. print_help() accepts a file kwarg that we can use to do this
parser.print_help()
async def up(self):
data = {"name": "postgres-data"}
# Create Postgres Volume
await self.docker_client.volumes.create(data)
# Create Walkoff encryption key
wek = await create_encryption_key(self.docker_client, "walkoff_encryption_key")
# Create internal user key
wik = await create_encryption_key(self.docker_client, "walkoff_internal_key")
# Create Postgres user password
wpk = await create_encryption_key(self.docker_client, "walkoff_postgres_key")
# Create Minio secret key
wmak = await create_encryption_key(self.docker_client, "walkoff_minio_access_key", b"walkoff")
wmsk = await create_encryption_key(self.docker_client, "walkoff_minio_secret_key")
# Set up a subcommand parser
parser = argparse.ArgumentParser(description="Bring the WALKOFF stack up and initialize it")
parser.add_argument("-b", "--build", action="store_true",
help="Builds and pushes all WALKOFF components to local registry.")
parser.add_argument("-d", "--debug", action="store_true",
help="Set log level to debug.")
parser.add_argument("-k", "--keys", action="store_true",
help="Prints all keys to STDOUT (dangerous).")
# Parse out the command
args = parser.parse_args(sys.argv[2:])
if args.debug:
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
logger.info("Creating persistent directories for registry, postgres, portainer...")
os.makedirs(static.REGISTRY_DATA_PATH, exist_ok=True)
os.makedirs(static.POSTGRES_DATA_PATH, exist_ok=True)
os.makedirs(static.PORTAINER_DATA_PATH, exist_ok=True)
os.makedirs(static.MINIO_DATA_PATH, exist_ok=True)
# Bring up the base compose with the registry
logger.info("Deploying base services (registry, postgres, portainer, redis)...")
base_compose = parse_yaml(config.BASE_COMPOSE)
await deploy_compose(base_compose)
await self.wait_for_registry()
# Merge the base, walkoff, and app composes
app_composes = generate_app_composes()
walkoff_compose = parse_yaml(config.WALKOFF_COMPOSE)
merged_compose = merge_composes(walkoff_compose, app_composes)
dump_yaml(config.TMP_COMPOSE, merged_compose)
if args.build:
walkoff_app_sdk = walkoff_compose["services"]["app_sdk"]
await build_image(self.docker_client, walkoff_app_sdk["image"],
walkoff_app_sdk["build"]["dockerfile"],
walkoff_app_sdk["build"]["context"],
self.dockerignore)
await push_image(self.docker_client, walkoff_app_sdk["image"])
builders = []
pushers = []
for service_name, service in walkoff_compose["services"].items():
if "build" in service:
build_func = build_image(self.docker_client, service["image"],
service["build"]["dockerfile"],
service["build"]["context"],
self.dockerignore)
push_func = push_image(self.docker_client, service["image"])
if args.debug:
await build_func
await push_func
else:
builders.append(build_func)
pushers.append(push_func)
if not args.debug:
logger.info("Building Docker images asynchronously, this could take some time...")
await asyncio.gather(*builders)
logger.info("Build process complete.")
logger.info("Pushing Docker images asynchronously, this could take some time...")
await asyncio.gather(*pushers)
logger.info("Push process complete.")
await self.wait_for_minio()
# await self.push_to_minio()
logger.info("Deploying Walkoff stack...")
return_code = await deploy_compose(merged_compose)
if args.keys:
if await are_you_sure("You specified -k/--keys, which will print all newly created keys to stdout."):
print(f"walkoff_encryption_key:\t\t{wek.decode()}")
print(f"walkoff_internal_key:\t\t{wik.decode()}")
print(f"walkoff_postgres_key:\t\t{wpk.decode()}")
print(f"walkoff_minio_access_key:\t{wmak.decode()}")
print(f"walkoff_minio_secret_key:\t{wmsk.decode()}\n\n")
logger.info("Walkoff stack deployed, it may take a little time to converge. \n"
"Use 'docker stack services walkoff' to check on Walkoff services. \n"
"Web interface should be available at 'https://127.0.0.1:8080' once walkoff_resource_nginx is up.")
return return_code
async def up(self):
# Set up a subcommand parser
parser = argparse.ArgumentParser(
description="Bring the WALKOFF stack up and initialize it")
parser.add_argument(
"-b",
"--build",
action="store_true",
help="Builds and pushes all WALKOFF components to local registry.")
parser.add_argument("-d",
"--debug",
action="store_true",
help="Set log level to debug.")
parser.add_argument("-y",
"--yes",
action="store_true",
help="Skips all verification questions.")
parser.add_argument("-r",
"--resources",
action="store_true",
help="Only runs the resource services.")
parser.add_argument("-o",
"--offline",
action="store_true",
help="Runs WALKOFF offline")
# Parse out the command
args = parser.parse_args(sys.argv[2:])
debug_pw = None
debug_pw_encryption = None
if args.debug:
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
if args.yes or await are_you_sure(
"You specified -d/--debug, which will use 'walkoff123456' as the password for all "
"resources, This should be used for debug only. "
"(Choose 'no' to use randomly generated passwords.)"):
debug_pw = b"walkoff123456"
debug_pw_encryption = b"walkoff123456789987654321walkoff"
# Create Walkoff encryption key
await create_encryption_key(self.docker_client, static.ENCRYPTION_KEY,
debug_pw_encryption)
# Create internal user key
await create_encryption_key(self.docker_client, static.INTERNAL_KEY,
debug_pw)
# Create Minio secret key
await create_encryption_key(self.docker_client,
static.MINIO_ACCESS_KEY, b"walkoff123456")
await create_encryption_key(self.docker_client,
static.MINIO_SECRET_KEY, debug_pw)
# Create Mongo user password
await create_encryption_key(self.docker_client, static.MONGO_KEY,
debug_pw)
# Create Redis key
await create_encryption_key(self.docker_client, static.REDIS_KEY,
debug_pw)
# Create volumes
logger.info(
"Creating volumes for persisting (registry, minio, mongo, portainer)..."
)
for volume in volume_names:
await self.docker_client.volumes.create({"name": volume})
# Bring up the base compose with the registry
base_compose = parse_yaml(config.BASE_COMPOSE)
for service_name, service in base_compose["services"].items():
if args.offline:
if not await image_exists(self.docker_client,
service["image"]):
if not await pull_image(self.docker_client,
service["image"]):
logger.info(
"Failed to pull requisite images, aborting deployment. "
"Please use the 'down' command to clean up. ")
return
else:
if not await pull_image(self.docker_client, service["image"]):
logger.info(
"Failed to pull requisite images, aborting deployment. "
"Please use the 'down' command to clean up. ")
return
logger.info(
"Deploying base services (registry, minio, mongo, portainer, redis)..."
)
await deploy_compose(base_compose)
if args.resources:
return
await self.wait_for_registry()
# Merge the base, walkoff, and app composes
app_composes = generate_app_composes()
walkoff_compose = parse_yaml(config.WALKOFF_COMPOSE)
merged_compose = merge_composes(walkoff_compose, app_composes)
dump_yaml(config.TMP_COMPOSE, merged_compose)
if args.build:
walkoff_app_sdk = walkoff_compose["services"]["app_sdk"]
await build_image(self.docker_client, walkoff_app_sdk["image"],
walkoff_app_sdk["build"]["dockerfile"],
walkoff_app_sdk["build"]["context"],
self.dockerignore)
await push_image(self.docker_client, walkoff_app_sdk["image"])
builders = []
pushers = []
for service_name, service in walkoff_compose["services"].items():
if "build" in service:
build_func = build_image(self.docker_client,
service["image"],
service["build"]["dockerfile"],
service["build"]["context"],
self.dockerignore)
push_func = push_image(self.docker_client,
service["image"])
if args.debug:
await build_func
await push_func
else:
builders.append(build_func)
pushers.append(push_func)
if not args.debug:
logger.info(
"Building Docker images asynchronously, this could take some time..."
)
await asyncio.gather(*builders)
logger.info("Build process complete.")
logger.info(
"Pushing Docker images asynchronously, this could take some time..."
)
await asyncio.gather(*pushers)
logger.info("Push process complete.")
await self.wait_for_minio()
# await self.push_to_minio()
logger.info("Deploying Walkoff stack...")
return_code = await deploy_compose(merged_compose)
logger.info(
"Walkoff stack deployed, it may take a little time to converge. \n"
"Use 'docker stack services walkoff' to check on Walkoff services. \n"
"Web interface should be available at 'https://127.0.0.1:8080' once walkoff_resource_nginx is up."
)
return return_code
async def down(self):
# Set up a subcommand parser
parser = argparse.ArgumentParser(
description=
"Remove the WALKOFF stack and optionally related artifacts.")
parser.add_argument("-k",
"--key",
action="store_true",
help="Removes the walkoff_encryption_key secret.")
parser.add_argument("-r",
"--registry",
action="store_true",
help="Clears the registry bind mount directory.")
parser.add_argument("-d",
"--debug",
action="store_true",
help="Set log level to debug.")
# Parse out the command
args = parser.parse_args(sys.argv[2:])
if args.debug:
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
logger.info("Removing Walkoff stack and related artifacts...")
proc = await asyncio.create_subprocess_exec(
"docker",
"stack",
"rm",
"walkoff",
stderr=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE)
await log_proc_output(proc)
# if not args.skipnetwork:
# logger.info("Waiting for containers to exit and network to be removed...")
# await exponential_wait(check_for_network, [self.docker_client], "Network walkoff_default still exists")
if args.key:
resp = input(
"Deleting encryption key will render database unreadable, and therefore it will be cleared. "
"This will delete all workflows, execution results, globals, users, roles, etc. "
"Are you sure? (yes/no): ")
while resp.lower() not in ("yes", "no"):
resp = input("Please answer 'yes' or 'no': ")
if resp.lower() == "yes":
await delete_encryption_key(self.docker_client,
"walkoff_encryption_key")
await delete_encryption_key(self.docker_client,
"walkoff_internal_key")
await delete_dir_contents("data/postgres")
if args.registry:
await delete_dir_contents("data/registry")
await delete_dir_contents("data/minio/min_data")
return proc.returncode
async def up(self):
# Create Walkoff encryption key
return_code = await create_encryption_key(self.docker_client,
"walkoff_encryption_key")
if return_code:
logger.exception(
"Could not create secret walkoff_encryption_key. Exiting.")
os._exit(return_code)
# Create internal user key
return_code2 = await create_encryption_key(self.docker_client,
"walkoff_internal_key")
if return_code2:
logger.exception(
"Could not create secret walkoff_internal_key. Exiting.")
os._exit(return_code2)
# Set up a subcommand parser
parser = argparse.ArgumentParser(
description="Bring the WALKOFF stack up and initialize it")
parser.add_argument(
"-b",
"--build",
action="store_true",
help="Builds and pushes all WALKOFF components to local registry.")
parser.add_argument("-d",
"--debug",
action="store_true",
help="Set log level to debug.")
# Parse out the command
args = parser.parse_args(sys.argv[2:])
if args.debug:
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
logger.info(
"Creating persistent directories for registry, postgres, portainer..."
)
os.makedirs(Path("data") / "registry" / "reg_data", exist_ok=True)
os.makedirs(Path("data") / "postgres" / "pg_data", exist_ok=True)
os.makedirs(Path("data") / "portainer" / "prt_data", exist_ok=True)
os.makedirs(Path("data") / "minio" / "min_data", exist_ok=True)
# Bring up the base compose with the registry
logger.info(
"Deploying base services (registry, postgres, portainer, redis)..."
)
base_compose = parse_yaml(config.BASE_COMPOSE)
await deploy_compose(base_compose)
await self.wait_for_registry()
# Merge the base, walkoff, and app composes
app_composes = generate_app_composes()
walkoff_compose = parse_yaml(config.WALKOFF_COMPOSE)
merged_compose = merge_composes(walkoff_compose, app_composes)
dump_yaml(config.TMP_COMPOSE, merged_compose)
if args.build:
walkoff_app_sdk = walkoff_compose["services"]["app_sdk"]
await build_image(self.docker_client, walkoff_app_sdk["image"],
walkoff_app_sdk["build"]["dockerfile"],
walkoff_app_sdk["build"]["context"],
self.dockerignore)
await push_image(self.docker_client, walkoff_app_sdk["image"])
for service_name, service in walkoff_compose["services"].items():
if "build" in service:
await build_image(self.docker_client, service["image"],
service["build"]["dockerfile"],
service["build"]["context"],
self.dockerignore)
await push_image(self.docker_client, service["image"])
await self.wait_for_minio()
await self.push_to_minio()
logger.info("Deploying Walkoff stack...")
return_code = await deploy_compose(merged_compose)
return return_code