def build(flavor_path: Tuple[str, ...], goal: Tuple[str, ...],
force_rebuild: bool, force_rebuild_from: Tuple[str, ...],
force_pull: bool, shortcut_build: bool, output_directory: str,
temporary_base_directory: str, log_build_context_content: bool,
cache_directory: str, source_docker_repository_name: str,
source_docker_tag_prefix: str, source_docker_username: str,
source_docker_password: str, target_docker_repository_name: str,
target_docker_tag_prefix: str, target_docker_username: str,
target_docker_password: str, workers: int,
task_dependencies_dot_file: str):
"""
This command builds all stages of the script language container flavor.
If stages are cached in a docker registry, they command is going to pull them,
instead of building them.
"""
import_build_steps(flavor_path)
set_build_config(force_rebuild, force_rebuild_from, force_pull,
log_build_context_content, output_directory,
temporary_base_directory, cache_directory)
set_docker_repository_config(source_docker_password,
source_docker_repository_name,
source_docker_username,
source_docker_tag_prefix, "source")
set_docker_repository_config(target_docker_password,
target_docker_repository_name,
target_docker_username,
target_docker_tag_prefix, "target")
tasks = lambda: [
DockerBuild(flavor_paths=list(flavor_path),
goals=list(goal),
shortcut_build=shortcut_build)
]
run_tasks(tasks, workers, task_dependencies_dot_file)
def push(flavor_path: Tuple[str, ...], goal: Tuple[str, ...], force_push: bool,
push_all: bool, force_rebuild: bool,
force_rebuild_from: Tuple[str, ...], force_pull: bool,
output_directory: str, temporary_base_directory: str,
log_build_context_content: bool, cache_directory: str,
source_docker_repository_name: str, source_docker_tag_prefix: str,
source_docker_username: str, source_docker_password: str,
target_docker_repository_name: str, target_docker_tag_prefix: str,
target_docker_username: str, target_docker_password: str,
workers: int, task_dependencies_dot_file: str):
"""
This command pushes all stages of the script language container flavor.
If the stages do not exists locally, the system will build or pull them before the push.
"""
import_build_steps(flavor_path)
set_build_config(force_rebuild, force_rebuild_from, force_pull,
log_build_context_content, output_directory,
temporary_base_directory, cache_directory)
set_docker_repository_config(source_docker_password,
source_docker_repository_name,
source_docker_username,
source_docker_tag_prefix, "source")
set_docker_repository_config(target_docker_password,
target_docker_repository_name,
target_docker_username,
target_docker_tag_prefix, "target")
tasks = lambda: [
DockerPush(force_push=force_push,
push_all=push_all,
goals=list(goal),
flavor_paths=list(flavor_path))
]
run_tasks(tasks, workers, task_dependencies_dot_file)
def spawn_test_environment(
environment_name: str,
database_port_forward: str,
bucketfs_port_forward: str,
output_directory: str,
temporary_base_directory: str,
workers: int,
task_dependencies_dot_file: str):
"""
This command spawn a test environment with a docker-db container and a conected test-container.
The test-container is reachable by the database for output redirects of udfs.
"""
set_build_config(False,
tuple(),
False,
False,
output_directory,
temporary_base_directory,
None,
None)
tasks = lambda: [SpawnTestDockerEnvironment(environment_name=environment_name,
database_port_forward=str(database_port_forward),
bucketfs_port_forward=str(bucketfs_port_forward))]
run_tasks(tasks, workers, task_dependencies_dot_file)
def upload(flavor_path: Tuple[str, ...], release_type: str, database_host: str,
bucketfs_port: int, bucketfs_username: str, bucketfs_password: str,
bucketfs_https: bool, bucketfs_name: str, bucket_name: str,
path_in_bucket: str, release_name: str, force_rebuild: bool,
force_rebuild_from: Tuple[str, ...], force_pull: bool,
output_directory: str, temporary_base_directory: str,
log_build_context_content: bool, cache_directory: str,
build_name: str, source_docker_repository_name: str,
source_docker_tag_prefix: str, source_docker_username: str,
source_docker_password: str, target_docker_repository_name: str,
target_docker_tag_prefix: str, target_docker_username: str,
target_docker_password: str, workers: int,
task_dependencies_dot_file: str):
"""
This command uploads the whole script language container package of the flavor to the database.
If the stages or the packaged container do not exists locally, the system will build, pull or
export them before the upload.
"""
import_build_steps(flavor_path)
set_build_config(force_rebuild, force_rebuild_from, force_pull,
log_build_context_content, output_directory,
temporary_base_directory, cache_directory, build_name)
set_docker_repository_config(source_docker_password,
source_docker_repository_name,
source_docker_username,
source_docker_tag_prefix, "source")
set_docker_repository_config(target_docker_password,
target_docker_repository_name,
target_docker_username,
target_docker_tag_prefix, "target")
if bucketfs_password is None:
bucketfs_password = getpass.getpass(
"BucketFS Password for BucketFS %s and User %s:" %
(bucketfs_name, bucketfs_username))
tasks = lambda: [
UploadContainer(flavor_paths=list(flavor_path),
release_types=list([release_type]),
database_host=database_host,
bucketfs_port=bucketfs_port,
bucketfs_username=bucketfs_username,
bucketfs_password=bucketfs_password,
bucket_name=bucket_name,
path_in_bucket=path_in_bucket,
bucketfs_https=bucketfs_https,
release_name=release_name,
bucketfs_name=bucketfs_name)
]
def on_success():
target = luigi.LocalTarget("%s/uploads/command_line_output" %
(output_directory))
with target.open("r") as f:
print(f.read())
run_tasks(tasks,
workers,
task_dependencies_dot_file,
on_success=on_success)
def clean_all_images(output_directory: str, docker_repository_name: str,
docker_tag_prefix: str, workers: int,
task_dependencies_dot_file: str):
"""
This command uploads the whole script language container package of the flavor to the database.
If the stages or the packaged container do not exists locally, the system will build, pull or
export them before the upload.
"""
set_output_directory(output_directory)
set_docker_repository_config(None, docker_repository_name, None,
docker_tag_prefix, "source")
set_docker_repository_config(None, docker_repository_name, None,
docker_tag_prefix, "target")
tasks = lambda: [CleanExaslcImages()]
run_tasks(tasks, workers, task_dependencies_dot_file)
def export(flavor_path: Tuple[str, ...], release_type: str, export_path: str,
release_name: str, force_rebuild: bool,
force_rebuild_from: Tuple[str, ...], force_pull: bool,
output_directory: str, temporary_base_directory: str,
log_build_context_content: bool, cache_directory: str,
build_name: str, source_docker_repository_name: str,
source_docker_tag_prefix: str, source_docker_username: str,
source_docker_password: str, target_docker_repository_name: str,
target_docker_tag_prefix: str, target_docker_username: str,
target_docker_password: str, workers: int,
task_dependencies_dot_file: str):
"""
This command exports the whole script language container package of the flavor,
ready for the upload into the bucketfs. If the stages do not exists locally,
the system will build or pull them before the exporting the packaged container.
"""
import_build_steps(flavor_path)
set_build_config(force_rebuild, force_rebuild_from, force_pull,
log_build_context_content, output_directory,
temporary_base_directory, cache_directory, build_name)
set_docker_repository_config(source_docker_password,
source_docker_repository_name,
source_docker_username,
source_docker_tag_prefix, "source")
set_docker_repository_config(target_docker_password,
target_docker_repository_name,
target_docker_username,
target_docker_tag_prefix, "target")
tasks = lambda: [
ExportContainers(flavor_paths=list(flavor_path),
release_types=list([release_type]),
export_path=export_path,
release_name=release_name)
]
def on_success():
with ExportContainers.command_line_output_target.open("r") as f:
print(f.read())
run_tasks(tasks,
workers,
task_dependencies_dot_file,
on_success=on_success)
def run_db_test(
flavor_path: Tuple[str, ...], release_type: str,
generic_language_test: Tuple[str, ...], test_folder: Tuple[str, ...],
test_file: Tuple[str,
...], test_language: Tuple[str,
...], test: Tuple[str,
...],
test_environment_vars: str, test_log_level: str, reuse_database: bool,
reuse_database_setup: bool, reuse_uploaded_container: bool,
force_rebuild: bool, force_rebuild_from: Tuple[str, ...],
force_pull: bool, output_directory: str, temporary_base_directory: str,
log_build_context_content: bool, cache_directory: str,
source_docker_repository_name: str, source_docker_tag_prefix: str,
source_docker_username: str, source_docker_password: str,
target_docker_repository_name: str, target_docker_tag_prefix: str,
target_docker_username: str, target_docker_password: str, workers: int,
task_dependencies_dot_file: str):
"""
This command runs the integration tests in local docker-db.
The systems spawns a test environment in which the test are executed.
After finishing the tests, the test environment gets cleaned up.
If the stages or the packaged container do not exists locally,
the system will build, pull or export them before running the tests.
"""
import_build_steps(flavor_path)
set_build_config(force_rebuild, force_rebuild_from, force_pull,
log_build_context_content, output_directory,
temporary_base_directory, cache_directory)
set_docker_repository_config(source_docker_password,
source_docker_repository_name,
source_docker_username,
source_docker_tag_prefix, "source")
set_docker_repository_config(target_docker_password,
target_docker_repository_name,
target_docker_username,
target_docker_tag_prefix, "target")
tasks = lambda: [
TestContainer(flavor_paths=list(flavor_path),
release_types=list([release_type]),
generic_language_tests=list(generic_language_test),
test_folders=list(test_folder),
test_files=list(test_file),
test_restrictions=list(test),
languages=list(test_language),
test_environment_vars=json.loads(test_environment_vars),
test_log_level=test_log_level,
reuse_database=reuse_database,
reuse_uploaded_container=reuse_uploaded_container,
reuse_database_setup=reuse_database_setup)
]
def on_success():
target = luigi.LocalTarget(
"%s/logs/test-runner/db-test/tests/current" % (output_directory))
print("Test Results:")
with target.open("r") as f:
print(f.read())
run_tasks(tasks,
workers,
task_dependencies_dot_file,
on_success=on_success)