def clean(self):
task = generate_root_task(
task_class=CleanImagesStartingWith,
starts_with_pattern="exasol-test-docker-build-base")
luigi.build([task], workers=1, local_scheduler=True, log_level="INFO")
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
def push_test_container(
force_push: bool, push_all: bool, force_rebuild: bool,
force_rebuild_from: Tuple[str, ...], force_pull: bool,
output_directory: str, temporary_base_directory: str,
log_build_context_content: bool, cache_directory: str, build_name: str,
source_docker_repository_name: str, source_docker_tag_prefix: str,
source_docker_username: str, source_docker_password: str,
target_docker_repository_name: str, target_docker_tag_prefix: str,
target_docker_username: str, target_docker_password: str, workers: int,
task_dependencies_dot_file: str):
"""
This command pushs all stages of the test container for the test environment.
If the stages do not exists locally, the system will build or pull them before the push.
"""
set_build_config(force_rebuild, force_rebuild_from, force_pull,
log_build_context_content, output_directory,
temporary_base_directory, cache_directory, build_name)
# Use AnalyzeTestContainer to ensure that all luigi processes got it loaded
analyze_task = AnalyzeTestContainer.__class__.__name__
set_docker_repository_config(source_docker_password,
source_docker_repository_name,
source_docker_username,
source_docker_tag_prefix, "source")
set_docker_repository_config(target_docker_password,
target_docker_repository_name,
target_docker_username,
target_docker_tag_prefix, "target")
task_creator = lambda: generate_root_task(task_class=
DockerTestContainerPush,
force_push=force_push,
push_all=push_all)
success, task = run_task(task_creator, workers, task_dependencies_dot_file)
if not success:
exit(1)
def test_json_pickle_parameter_success(self):
task = generate_root_task(task_class=TestTask9_Success)
try:
luigi.build([task],
workers=3,
local_scheduler=True,
log_level="INFO")
finally:
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
def test_json_pickle_parameter_fail(self):
task = generate_root_task(task_class=TestTask9_Fail)
try:
with self.assertRaises(Exception) as context:
luigi.build([task],
workers=3,
local_scheduler=True,
log_level="INFO")
print(context.exception)
finally:
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
def test_default_parameter(self):
task = generate_root_task(task_class=TestDockerBuildBase)
try:
luigi.build([task],
workers=1,
local_scheduler=True,
log_level="INFO")
self.assert_image_exists(
"exasol-test-docker-build-base:test-analyze-image-1")
finally:
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
def run1(self):
task = generate_root_task(task_class=TestTask, reuse=False, attempt=1)
try:
success = luigi.build([task], workers=1, local_scheduler=True, log_level="INFO")
if success:
result = task.get_return_object()
task.cleanup(True)
return result
else:
raise Exception("Task failed")
except Exception as e:
task.cleanup(False)
raise RuntimeError("Error spawning test environment") from e
def _run_it(self, different_grandchild, use_dynamic_dependency, expected_result):
global global_counter
global_counter = 0
task = generate_root_task(task_class=TestTaskBase,
different_grandchild=different_grandchild,
use_dynamic_dependency=use_dynamic_dependency)
try:
luigi.build([task], workers=3, local_scheduler=True, log_level="INFO")
task.cleanup(success=True)
finally:
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
self.assertEqual(global_counter, expected_result, "number of Cleanups not matching")
def test_collect_failures_diffrent_task_fail(self):
task = generate_root_task(task_class=TestTask11)
try:
luigi.build([task],
workers=5,
local_scheduler=True,
log_level="INFO")
failures = task.collect_failures()
print()
print("Collected Failures:")
for failure in failures:
print(failure)
print()
print()
self.assertGreater(len(failures), 1)
finally:
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
def test_collect_failures_one_task_fails_but_is_dependency_of_multiple(
self):
task = generate_root_task(task_class=TestTask13)
try:
luigi.build([task],
workers=3,
local_scheduler=True,
log_level="INFO")
failures = task.collect_failures()
print()
print("Collected Failures:")
for failure in failures:
print(failure)
print()
print()
self.assertEquals(len(failures), 1)
finally:
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
def build_test_container(
force_rebuild: bool,
force_rebuild_from: Tuple[str, ...],
force_pull: bool,
output_directory: str,
temporary_base_directory: str,
log_build_context_content: bool,
cache_directory: str,
build_name: str,
source_docker_repository_name: str,
source_docker_tag_prefix: str,
source_docker_username: str,
source_docker_password: str,
target_docker_repository_name: str,
target_docker_tag_prefix: str,
target_docker_username: str,
target_docker_password: str,
workers: int,
task_dependencies_dot_file: str):
"""
This command builds all stages of the test container for the test environment.
If stages are cached in a docker registry, they command is going to pull them,
instead of building them.
"""
set_build_config(force_rebuild,
force_rebuild_from,
force_pull,
log_build_context_content,
output_directory,
temporary_base_directory,
cache_directory,
build_name)
# Use AnalyzeTestContainer to ensure that all luigi processes got it loaded
analyze_task = AnalyzeTestContainer.__class__.__name__
set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username,
source_docker_tag_prefix, "source")
set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username,
target_docker_tag_prefix, "target")
task_creator = lambda: generate_root_task(task_class=DockerTestContainerBuild)
success, task = run_task(task_creator, workers, task_dependencies_dot_file)
if not success:
exit(1)
def run_test_different_logging_file_raises_error() -> None:
"""
Integration test which verifies that changing the log path from one invocation of run_task to the next will raise
an error.
"""
with tempfile.TemporaryDirectory() as temp_dir:
task_creator = lambda: generate_root_task(task_class=TestTask,
x="Test")
success, task = run_task(task_creator,
workers=5,
task_dependencies_dot_file=None)
assert success
log_path = Path(temp_dir) / "main.log"
os.environ[LOG_ENV_VARIABLE_NAME] = str(log_path)
success, task = run_task(task_creator,
workers=5,
task_dependencies_dot_file=None)
assert success is False
def run_spawn_test_env(self, cleanup: bool):
result = None
task = generate_root_task(
task_class=SpawnTestEnvironment,
reuse_database_setup=True,
reuse_database=True,
reuse_test_container=True,
no_test_container_cleanup_after_success=not cleanup,
no_database_cleanup_after_success=not cleanup,
external_exasol_xmlrpc_host="",
external_exasol_db_host="",
external_exasol_xmlrpc_port=0,
external_exasol_db_user="",
external_exasol_db_password="",
external_exasol_xmlrpc_user="",
external_exasol_xmlrpc_password="",
external_exasol_xmlrpc_cluster_name="",
external_exasol_bucketfs_write_password="",
environment_type=EnvironmentType.docker_db,
environment_name=self.env_name(),
docker_db_image_version=self.docker_db_version_parameter,
docker_db_image_name="exasol/docker-db")
try:
success = luigi.build([task],
workers=1,
local_scheduler=True,
log_level="INFO")
if success:
result = task
else:
raise Exception("Task failed")
except Exception as e:
task.cleanup(False)
raise RuntimeError("Error spawning test environment") from e
return result
def create_task():
return generate_root_task(task_class=TestTask,
x=f"{next(task_id_generator)}")
def test_failing_task(self):
task = generate_root_task(task_class=TestTask4)
luigi.build([task], workers=1, local_scheduler=True, log_level="INFO")
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
def clean(docker_repository_name):
task = generate_root_task(task_class=CleanImagesStartingWith, starts_with_pattern=docker_repository_name)
luigi.build([task], workers=1, local_scheduler=True, log_level="INFO")
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
def test_non_valid_non_default_goal(self):
with self.assertRaises(Exception) as contex:
task = generate_root_task(task_class=TestDockerBuildBase,
goals=["test-analyze-image-3"])
self.assertIn("Unknown goal(s)", str(contex.exception))
def spawn_test_environment(
environment_name: str,
database_port_forward: int,
bucketfs_port_forward: int,
db_mem_size: str,
db_disk_size: str,
nameserver:List[str],
deactivate_database_setup: bool,
docker_runtime: str,
docker_db_image_version: str,
docker_db_image_name: str,
source_docker_repository_name: str,
source_docker_tag_prefix: str,
source_docker_username: str,
source_docker_password: str,
target_docker_repository_name: str,
target_docker_tag_prefix: str,
target_docker_username: str,
target_docker_password: str,
output_directory: str,
temporary_base_directory: str,
workers: int,
task_dependencies_dot_file: str,
create_certificates: bool):
"""
This command spawn a test environment with a docker-db container and a connected test-container.
The test-container is reachable by the database for output redirects of UDFs.
"""
parsed_db_mem_size = humanfriendly.parse_size(db_mem_size)
if parsed_db_mem_size < humanfriendly.parse_size("1 GiB"):
print("The --db-mem-size needs to be at least 1 GiB.")
exit(1)
parsed_db_disk_size = humanfriendly.parse_size(db_disk_size)
if parsed_db_disk_size < humanfriendly.parse_size("100 MiB"):
print("The --db-disk-size needs to be at least 100 MiB.")
exit(1)
set_build_config(False,
tuple(),
False,
False,
output_directory,
temporary_base_directory,
None,
None)
set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username,
source_docker_tag_prefix, "source")
set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username,
target_docker_tag_prefix, "target")
task_creator = lambda: generate_root_task(task_class=SpawnTestEnvironmentWithDockerDB,
environment_name=environment_name,
database_port_forward=str(database_port_forward) if database_port_forward is not None else None,
bucketfs_port_forward=str(bucketfs_port_forward) if bucketfs_port_forward is not None else None,
mem_size=db_mem_size,
disk_size=db_disk_size,
nameservers=nameserver,
docker_runtime=docker_runtime,
docker_db_image_version=docker_db_image_version,
docker_db_image_name=docker_db_image_name,
db_user="******",
db_password="******",
bucketfs_write_password="******",
no_test_container_cleanup_after_success=True,
no_test_container_cleanup_after_failure=False,
no_database_cleanup_after_success=True,
no_database_cleanup_after_failure=False,
is_setup_database_activated=not deactivate_database_setup,
create_certificates=create_certificates
)
success, task = run_task(task_creator, workers, task_dependencies_dot_file)
if not success:
exit(1)
def test_common_parameter(self):
task = generate_root_task(task_class=TestTask7, test_parameter="input")
luigi.build([task], workers=1, local_scheduler=True, log_level="INFO")
if task._get_tmp_path_for_job().exists():
shutil.rmtree(str(task._get_tmp_path_for_job()))
