class DockerImageCreatorBaseTask(DockerBaseTask):
image_name = luigi.Parameter()
# ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown
image_info = JsonPickleParameter(
ImageInfo,
visibility=luigi.parameter.ParameterVisibility.HIDDEN,
significant=True) # type:ImageInfo
class DockerCreateImageTask(DockerBaseTask):
image_name = luigi.Parameter()
# ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown
image_info = JsonPickleParameter(ImageInfo,
visibility=luigi.parameter.ParameterVisibility.HIDDEN,
significant=True) # type: ImageInfo
def run_task(self):
new_image_info = yield from self.build(self.image_info)
self.return_object(new_image_info)
def build(self, image_info: ImageInfo):
if image_info.image_state == ImageState.NEEDS_TO_BE_BUILD.name:
task = self.create_child_task(DockerBuildImageTask,
image_name=self.image_name,
image_info=image_info)
yield from self.run_dependencies(task)
image_info.image_state = ImageState.WAS_BUILD.name # TODO clone and change
return image_info
elif image_info.image_state == ImageState.CAN_BE_LOADED.name:
task = self.create_child_task(DockerLoadImageTask,
image_name=self.image_name,
image_info=image_info)
yield from self.run_dependencies(task)
image_info.image_state = ImageState.WAS_LOADED.name
return image_info
elif image_info.image_state == ImageState.REMOTE_AVAILABLE.name:
task = self.create_child_task(DockerPullImageTask,
image_name=self.image_name,
image_info=image_info)
yield from self.run_dependencies(task)
image_info.image_state = ImageState.WAS_PULLED.name
return image_info
elif image_info.image_state == ImageState.TARGET_LOCALLY_AVAILABLE.name:
image_info.image_state = ImageState.USED_LOCAL.name
return image_info
elif image_info.image_state == ImageState.SOURCE_LOCALLY_AVAILABLE.name:
image_info.image_state = ImageState.WAS_TAGED.name
self.rename_source_image_to_target_image(image_info)
return image_info
else:
raise Exception("Task %s: Image state %s not supported for image %s",
self.task_id, image_info.image_state, image_info.get_target_complete_name())
def rename_source_image_to_target_image(self, image_info):
with self._get_docker_client() as docker_client:
docker_client.images.get(image_info.get_source_complete_name()).tag(
repository=image_info.target_repository_name,
tag=image_info.get_target_complete_tag()
)
class DockerSaveImageTask(DockerSaveImageBaseTask):
# We need to create the DockerCreateImageTask for DockerSaveImageTask dynamically,
# because we want to save as soon as possible after an image was build and
# don't want to wait for the save finishing before starting to build depended images,
# but we also need to create a DockerSaveImageTask for each DockerCreateImageTask of a goal
required_task_info = JsonPickleParameter(
RequiredTaskInfo,
visibility=luigi.parameter.ParameterVisibility.HIDDEN,
significant=True) # type: RequiredTaskInfo
def get_docker_image_task(self):
module = importlib.import_module(self.required_task_info.module_name)
class_ = getattr(module, self.required_task_info.class_name)
instance = self.create_child_task(class_,
**self.required_task_info.params)
return instance
class PopulateEngineSmallTestDataToDatabase(DockerBaseTask,
DatabaseCredentialsParameter):
logger = logging.getLogger('luigi-interface')
environment_name = luigi.Parameter()
reuse_data = luigi.BoolParameter(False, significant=False)
test_environment_info = JsonPickleParameter(
EnvironmentInfo, significant=False) # type: EnvironmentInfo
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._test_container_info = self.test_environment_info.test_container_info
self._database_info = self.test_environment_info.database_info
def run_task(self):
if self.reuse_data and self._database_info.reused:
self.logger.warning("Reusing data")
self.write_logs("Reused")
else:
self.populate_data()
def populate_data(self):
self.logger.warning("Uploading data")
username = self.db_user
password = self.db_password
with self._get_docker_client() as docker_client:
test_container = docker_client.containers.get(
self._test_container_info.container_name)
cmd = f"""cd /tests/test/enginedb_small; $EXAPLUS -c '{self._database_info.host}:{self._database_info.db_port}' -u '{username}' -p '{password}' -f import.sql -jdbcparam 'validateservercertificate=0'"""
bash_cmd = f"""bash -c "{cmd}" """
exit_code, output = test_container.exec_run(cmd=bash_cmd)
self.write_logs(output.decode("utf-8"))
if exit_code != 0:
raise Exception(
"Failed to populate the database with data.\nLog: %s" % cmd +
"\n" + output.decode("utf-8"))
def write_logs(self, output: str):
log_file = Path(self.get_log_path(), "log")
with log_file.open("w") as file:
file.write(output)
class DockerCreateImageTaskWithDeps(DockerCreateImageTask):
# ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown
required_task_infos = JsonPickleParameter(RequiredTaskInfoDict,
visibility=luigi.parameter.ParameterVisibility.HIDDEN,
significant=True) # type: RequiredTaskInfoDict
def register_required(self):
self.required_tasks = {key: self.create_required_task(required_task_info)
for key, required_task_info
in self.required_task_infos.infos.items()}
self.futures = self.register_dependencies(self.required_tasks)
def create_required_task(self, required_task_info: RequiredTaskInfo) -> DockerCreateImageTask:
module = importlib.import_module(required_task_info.module_name)
class_ = getattr(module, required_task_info.class_name)
instance = self.create_child_task(class_, **required_task_info.params)
return instance
def run_task(self):
image_infos = self.get_values_from_futures(self.futures)
image_info = copy.copy(self.image_info)
image_info.depends_on_images = image_infos
new_image_info = yield from self.build(image_info)
self.return_object(new_image_info)
class SpawnTestDockerDatabase(DockerBaseTask,
DockerDBTestEnvironmentParameter):
environment_name = luigi.Parameter() # type: str
db_container_name = luigi.Parameter() # type: str
attempt = luigi.IntParameter(1) # type: int
network_info = JsonPickleParameter(
DockerNetworkInfo, significant=False) # type: DockerNetworkInfo
ip_address_index_in_subnet = luigi.IntParameter(
significant=False) # type: int
docker_runtime = luigi.OptionalParameter(None,
significant=False) # type: str
certificate_volume_name = luigi.OptionalParameter(None, significant=False)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.ip_address_index_in_subnet < 0:
raise Exception(
"ip_address_index_in_subnet needs to be greater than 0 got %s"
% self.ip_address_index_in_subnet)
self.db_version = DbVersion.from_db_version_str(
self.docker_db_image_version)
self.docker_db_config_resource_name = f"docker_db_config/{self.db_version}"
def run_task(self):
subnet = netaddr.IPNetwork(self.network_info.subnet)
db_ip_address = str(subnet[2 + self.ip_address_index_in_subnet])
db_private_network = "{ip}/{prefix}".format(ip=db_ip_address,
prefix=subnet.prefixlen)
database_info = None
if self.network_info.reused:
database_info = self._try_to_reuse_database(db_ip_address)
if database_info is None:
database_info = self._create_database_container(
db_ip_address, db_private_network)
self.return_object(database_info)
def _try_to_reuse_database(self, db_ip_address: str) -> DatabaseInfo:
self.logger.info("Try to reuse database container %s",
self.db_container_name)
database_info = None
try:
database_info = self._create_database_info(
db_ip_address=db_ip_address, reused=True)
except Exception as e:
self.logger.warning(
"Tried to reuse database container %s, but got Exeception %s. "
"Fallback to create new database.", self.db_container_name, e)
return database_info
def _handle_output(self, output_generator, image_info: ImageInfo):
log_file_path = self.get_log_path().joinpath(
"pull_docker_db_image.log")
with PullLogHandler(log_file_path, self.logger,
image_info) as log_hanlder:
still_running_logger = StillRunningLogger(
self.logger,
"pull image %s" % image_info.get_source_complete_name())
for log_line in output_generator:
still_running_logger.log()
log_hanlder.handle_log_lines(log_line)
def _create_database_container(self, db_ip_address: str,
db_private_network: str):
self.logger.info("Starting database container %s",
self.db_container_name)
with self._get_docker_client() as docker_client:
try:
docker_client.containers.get(self.db_container_name).remove(
force=True, v=True)
except:
pass
docker_db_image_info = self._pull_docker_db_images_if_necassary()
db_volume = self._prepare_db_volume(docker_client,
db_private_network,
docker_db_image_info)
ports = {}
if self.database_port_forward is not None:
ports[f"{DB_PORT}/tcp"] = ('0.0.0.0',
int(self.database_port_forward))
if self.bucketfs_port_forward is not None:
ports[f"{BUCKETFS_PORT}/tcp"] = (
'0.0.0.0', int(self.bucketfs_port_forward))
volumes = {db_volume.name: {"bind": "/exa", "mode": "rw"}}
if self.certificate_volume_name is not None:
volumes[self.certificate_volume_name] = {
"bind": CERTIFICATES_MOUNT_DIR,
"mode": "ro"
}
db_container = \
docker_client.containers.create(
image="%s" % (docker_db_image_info.get_source_complete_name()),
name=self.db_container_name,
detach=True,
privileged=True,
volumes=volumes,
network_mode=None,
ports=ports,
runtime=self.docker_runtime
)
docker_network = docker_client.networks.get(
self.network_info.network_name)
network_aliases = self._get_network_aliases()
docker_network.connect(db_container,
ipv4_address=db_ip_address,
aliases=network_aliases)
db_container.start()
database_info = self._create_database_info(
db_ip_address=db_ip_address, reused=False)
return database_info
def _get_network_aliases(self):
network_aliases = [
"exasol_test_database", "exasol-test-database",
self.db_container_name
]
return network_aliases
def _create_database_info(self, db_ip_address: str,
reused: bool) -> DatabaseInfo:
with self._get_docker_client() as docker_client:
db_container = docker_client.containers.get(self.db_container_name)
if db_container.status != "running":
raise Exception(
f"Container {self.db_container_name} not running")
network_aliases = self._get_network_aliases()
container_info = \
ContainerInfo(container_name=self.db_container_name,
ip_address=db_ip_address,
network_aliases=network_aliases,
network_info=self.network_info,
volume_name=self._get_db_volume_name())
database_info = \
DatabaseInfo(host=db_ip_address, db_port=DB_PORT, bucketfs_port=BUCKETFS_PORT,
reused=reused, container_info=container_info)
return database_info
def _pull_docker_db_images_if_necassary(self):
image_name = "exasol/docker-db"
docker_db_image_info = ImageInfo(
target_repository_name=image_name,
source_repository_name=image_name,
source_tag=self.docker_db_image_version,
target_tag=self.docker_db_image_version,
hash_value="",
commit="",
image_description=None)
with self._get_docker_client() as docker_client:
try:
docker_client.images.get(
docker_db_image_info.get_source_complete_name())
except docker.errors.ImageNotFound as e:
self.logger.info(
"Pulling docker-db image %s",
docker_db_image_info.get_source_complete_name())
output_generator = docker_client.api.pull(
docker_db_image_info.source_repository_name,
tag=docker_db_image_info.source_tag,
stream=True)
self._handle_output(output_generator, docker_db_image_info)
return docker_db_image_info
def _prepare_db_volume(self, docker_client, db_private_network: str,
docker_db_image_info: ImageInfo) -> Volume:
db_volume_name = self._get_db_volume_name()
db_volume_preperation_container_name = self._get_db_volume_preperation_container_name(
)
self._remove_container(db_volume_preperation_container_name)
self._remove_volume(db_volume_name)
db_volume, volume_preparation_container = \
volume_preparation_container, volume_preparation_container = \
self._create_volume_and_container(docker_client, db_volume_name,
db_volume_preperation_container_name)
try:
self._upload_init_db_files(volume_preparation_container,
db_private_network)
self._execute_init_db(db_volume, volume_preparation_container)
return db_volume
finally:
volume_preparation_container.remove(force=True)
def _get_db_volume_preperation_container_name(self):
db_volume_preperation_container_name = f"""{self.db_container_name}_preparation"""
return db_volume_preperation_container_name
def _get_db_volume_name(self):
db_volume_name = f"""{self.db_container_name}_volume"""
return db_volume_name
def _remove_container(self, db_volume_preperation_container_name):
try:
with self._get_docker_client() as docker_client:
docker_client.containers.get(
db_volume_preperation_container_name).remove(force=True)
self.logger.info("Removed container %s",
db_volume_preperation_container_name)
except docker.errors.NotFound:
pass
def _remove_volume(self, db_volume_name):
try:
with self._get_docker_client() as docker_client:
docker_client.volumes.get(db_volume_name).remove(force=True)
self.logger.info("Removed volume %s", db_volume_name)
except docker.errors.NotFound:
pass
def _create_volume_and_container(self, docker_client, db_volume_name, db_volume_preperation_container_name) \
-> Tuple[Volume, Container]:
db_volume = docker_client.volumes.create(db_volume_name)
volume_preparation_container = \
docker_client.containers.run(
image="ubuntu:18.04",
name=db_volume_preperation_container_name,
auto_remove=True,
command="sleep infinity",
detach=True,
volumes={
db_volume.name: {"bind": "/exa", "mode": "rw"}},
labels={"test_environment_name": self.environment_name, "container_type": "db_container"})
return db_volume, volume_preparation_container
def _upload_init_db_files(self, volume_preperation_container: Container,
db_private_network: str):
copy = DockerContainerCopy(volume_preperation_container)
init_db_script_str = pkg_resources.resource_string(
PACKAGE_NAME,
f"{self.docker_db_config_resource_name}/init_db.sh") # type: bytes
copy.add_string_to_file("init_db.sh",
init_db_script_str.decode("utf-8"))
self._add_exa_conf(copy, db_private_network)
copy.copy("/")
def _add_exa_conf(self, copy: DockerContainerCopy,
db_private_network: str):
certificate_dir = CERTIFICATES_MOUNT_DIR if self.certificate_volume_name is not None \
else CERTIFICATES_DEFAULT_DIR
template_str = pkg_resources.resource_string(
PACKAGE_NAME,
f"{self.docker_db_config_resource_name}/EXAConf") # type: bytes
template = Template(template_str.decode("utf-8"))
rendered_template = template.render(
private_network=db_private_network,
db_version=str(self.db_version),
image_version=self.docker_db_image_version,
mem_size=self.mem_size,
disk_size=self.disk_size,
name_servers=",".join(self.nameservers),
certificate_dir=certificate_dir)
copy.add_string_to_file("EXAConf", rendered_template)
def _execute_init_db(self, db_volume: Volume,
volume_preperation_container: Container):
disk_size_in_bytes = humanfriendly.parse_size(self.disk_size)
min_overhead_in_gigabyte = 2 # Exasol needs at least a 2 GB larger device than the configured disk size
overhead_factor = max(0.01,
(min_overhead_in_gigabyte * 1024 * 1024 * 1024) /
disk_size_in_bytes) # and in general 1% larger
device_size_in_bytes = disk_size_in_bytes * (1 + overhead_factor)
device_size_in_megabytes = math.ceil(
device_size_in_bytes /
(1024 *
1024)) # The init_db.sh script works with MB, because its faster
self.logger.info(
f"Creating database volume of size {device_size_in_megabytes / 1024} GB using and overhead factor of {overhead_factor}"
)
(exit_code, output) = volume_preperation_container.exec_run(
cmd=f"bash /init_db.sh {device_size_in_megabytes}")
if exit_code != 0:
raise Exception(
"Error during preperation of docker-db volume %s got following output %s"
% (db_volume.name, output))
def cleanup_task(self, success):
if (success and not self.no_database_cleanup_after_success) or \
(not success and not self.no_database_cleanup_after_failure):
db_volume_preperation_container_name = self._get_db_volume_preperation_container_name(
)
try:
self.logger.info(f"Cleaning up container %s",
db_volume_preperation_container_name)
self._remove_container(db_volume_preperation_container_name)
except Exception as e:
self.logger.error(f"Error during removing container %s: %s",
db_volume_preperation_container_name, e)
try:
self.logger.info(f"Cleaning up container %s",
self.db_container_name)
self._remove_container(self.db_container_name)
except Exception as e:
self.logger.error(f"Error during removing container %s: %s",
self.db_container_name, e)
db_volume_name = self._get_db_volume_name()
try:
self.logger.info(f"Cleaning up docker volumne %s",
db_volume_name)
self._remove_volume(db_volume_name)
except Exception as e:
self.logger.error(
f"Error during removing docker volume %s: %s",
db_volume_name, e)
class WaitForTestExternalDatabase(DockerBaseTask,
DatabaseCredentialsParameter):
environment_name = luigi.Parameter()
test_container_info = JsonPickleParameter(ContainerInfo, significant=False) # type: ContainerInfo
database_info = JsonPickleParameter(DatabaseInfo, significant=False) # type: DatabaseInfo
db_startup_timeout_in_seconds = luigi.IntParameter(1 * 60, significant=False)
attempt = luigi.IntParameter(1)
def run_task(self):
with self._get_docker_client() as docker_client:
test_container = docker_client.containers.get(self.test_container_info.container_name)
is_database_ready = self.wait_for_database_startup(test_container)
self.return_object(is_database_ready)
def wait_for_database_startup(self, test_container: Container):
is_database_ready_thread = self.start_wait_threads(test_container)
is_database_ready = self.wait_for_threads(is_database_ready_thread)
self.join_threads(is_database_ready_thread)
return is_database_ready
def start_wait_threads(self, test_container):
is_database_ready_thread = IsDatabaseReadyThread(self.logger,
self.database_info,
self.get_database_credentials(),
test_container)
is_database_ready_thread.start()
return is_database_ready_thread
def join_threads(self, is_database_ready_thread: IsDatabaseReadyThread):
is_database_ready_thread.stop()
is_database_ready_thread.join()
def wait_for_threads(self, is_database_ready_thread: IsDatabaseReadyThread):
is_database_ready = False
reason = None
start_time = datetime.now()
while (True):
if is_database_ready_thread.finish:
is_database_ready = True
break
if self.timeout_occured(start_time):
reason = f"timeout after after {self.db_startup_timeout_in_seconds} seconds"
is_database_ready = False
break
time.sleep(1)
if not is_database_ready:
self.log_database_not_ready(is_database_ready_thread, reason)
is_database_ready_thread.stop()
return is_database_ready
def log_database_not_ready(self, is_database_ready_thread, reason):
log_information = f"""
========== IsDatabaseReadyThread output db connection: ============
{is_database_ready_thread.output_db_connection}
========== IsDatabaseReadyThread output bucketfs connection: ============
{is_database_ready_thread.output_bucketfs_connection}
"""
self.logger.warning(
'Database startup failed for following reason "%s", here some debug information \n%s',
reason, log_information)
def timeout_occured(self, start_time):
timeout = timedelta(seconds=self.db_startup_timeout_in_seconds)
return datetime.now() - start_time > timeout
class SetupExternalDatabaseHost(DependencyLoggerBaseTask,
ExternalDatabaseXMLRPCParameter,
ExternalDatabaseHostParameter,
DatabaseCredentialsParameter):
environment_name = luigi.Parameter()
network_info = JsonPickleParameter(
DockerNetworkInfo, significant=False) # type: DockerNetworkInfo
attempt = luigi.IntParameter(1)
def run_task(self):
database_host = self.external_exasol_db_host
if self.external_exasol_db_host == "localhost" or \
self.external_exasol_db_host == "127.0.01":
database_host = self.network_info.gateway
self.setup_database()
database_info = DatabaseInfo(
host=database_host,
db_port=self.external_exasol_db_port,
bucketfs_port=self.external_exasol_bucketfs_port,
reused=False)
self.return_object(database_info)
def setup_database(self):
if self.external_exasol_xmlrpc_host is not None:
# TODO add option to use unverified ssl
cluster = self.get_xml_rpc_object()
self.start_database(cluster)
cluster.bfsdefault.editBucketFS(
{'http_port': int(self.external_exasol_bucketfs_port)})
try:
cluster.bfsdefault.addBucket({
'bucket_name':
'myudfs',
'public_bucket':
True,
'read_password':
self.bucketfs_write_password,
'write_password':
self.bucketfs_write_password
})
except Exception as e:
self.logger.info(e)
try:
cluster.bfsdefault.addBucket({
'bucket_name':
'jdbc_adapter',
'public_bucket':
True,
'read_password':
self.bucketfs_write_password,
'write_password':
self.bucketfs_write_password
})
except Exception as e:
self.logger.info(e)
def get_xml_rpc_object(self, object_name: str = ""):
uri = 'https://{user}:{password}@{host}:{port}/{cluster_name}/{object_name}'.format(
user=quote_plus(self.external_exasol_xmlrpc_user),
password=quote_plus(self.external_exasol_xmlrpc_password),
host=self.external_exasol_xmlrpc_host,
port=self.external_exasol_xmlrpc_port,
cluster_name=self.external_exasol_xmlrpc_cluster_name,
object_name=object_name)
server = ServerProxy(uri, context=ssl._create_unverified_context())
return server
def start_database(self, cluster: ServerProxy):
storage = self.get_xml_rpc_object("storage")
# wait until all nodes are online
self.logger.info('Waiting until all nodes are online')
all_nodes_online = False
while not all_nodes_online:
all_nodes_online = True
for nodeName in cluster.getNodeList():
node_state = self.get_xml_rpc_object(nodeName).getNodeState()
if node_state['status'] != 'Running':
all_nodes_online = False
break
sleep(5)
self.logger.info('All nodes are online now')
# start EXAStorage
if not storage.serviceIsOnline():
if storage.startEXAStorage() != 'OK':
self.logger.info('EXAStorage has been started successfully')
else:
raise Exception('Not able startup EXAStorage!\n')
elif storage.serviceIsOnline():
self.logger.info(
'EXAStorage already online; continuing startup process')
# triggering database startup
for databaseName in cluster.getDatabaseList():
database = self.get_xml_rpc_object('/db_' +
quote_plus(databaseName))
if not database.runningDatabase():
self.logger.info('Starting database instance %s' %
databaseName)
database.startDatabase()
else:
self.logger.info('Database instance %s already running' %
databaseName)
class UploadFileToBucketFS(DockerBaseTask):
environment_name = luigi.Parameter()
test_environment_info = JsonPickleParameter(
EnvironmentInfo, significant=False) # type: EnvironmentInfo
reuse_uploaded = luigi.BoolParameter(False, significant=False)
bucketfs_write_password = luigi.Parameter(
significant=False,
visibility=luigi.parameter.ParameterVisibility.HIDDEN)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._test_container_info = self.test_environment_info.test_container_info
self._database_info = self.test_environment_info.database_info
def run_task(self):
file_to_upload = self.get_file_to_upload()
upload_target = self.get_upload_target()
pattern_to_wait_for = self.get_pattern_to_wait_for()
log_file = self.get_log_file()
sync_time_estimation = self.get_sync_time_estimation()
with self._get_docker_client() as docker_client:
if self._database_info.container_info is not None:
database_container = docker_client.containers.get(
self._database_info.container_info.container_name)
else:
database_container = None
if not self.should_be_reused(upload_target):
self.upload_and_wait(database_container, file_to_upload,
upload_target, log_file,
pattern_to_wait_for, sync_time_estimation)
else:
self.logger.warning(
"Reusing uploaded target %s instead of file %s",
upload_target, file_to_upload)
self.write_logs("Reusing")
def upload_and_wait(self, database_container, file_to_upload: str,
upload_target: str, log_file: str,
pattern_to_wait_for: str, sync_time_estimation: int):
still_running_logger = StillRunningLogger(
self.logger,
"file upload of %s to %s" % (file_to_upload, upload_target))
thread = StillRunningLoggerThread(still_running_logger)
thread.start()
sync_checker = self.get_sync_checker(database_container,
sync_time_estimation, log_file,
pattern_to_wait_for)
sync_checker.prepare_upload()
output = self.upload_file(file_to_upload=file_to_upload,
upload_target=upload_target)
sync_checker.wait_for_bucketfs_sync()
thread.stop()
thread.join()
self.write_logs(output)
def get_sync_checker(self, database_container: Container,
sync_time_estimation: int, log_file: str,
pattern_to_wait_for: str):
if database_container is not None:
return DockerDBLogBasedBucketFSSyncChecker(
database_container=database_container,
log_file_to_check=log_file,
pattern_to_wait_for=pattern_to_wait_for,
logger=self.logger,
bucketfs_write_password=self.bucketfs_write_password)
else:
return TimeBasedBucketFSSyncWaiter(sync_time_estimation)
def should_be_reused(self, upload_target: str):
return self.reuse_uploaded and self.exist_file_in_bucketfs(
upload_target)
def exist_file_in_bucketfs(self, upload_target: str) -> bool:
self.logger.info("Check if file %s exist in bucketfs", upload_target)
command = self.generate_list_command(upload_target)
exit_code, log_output = self.run_command("list", command)
if exit_code != 0:
self.write_logs(log_output)
raise Exception(
"List files in bucketfs failed, got following output %s" %
(log_output))
upload_target_in_bucket = "/".join(upload_target.split("/")[1:])
if upload_target_in_bucket in log_output.splitlines():
return True
else:
return False
def generate_list_command(self, upload_target: str):
bucket = upload_target.split("/")[0]
url = "http://*****:*****@{host}:{port}/{bucket}".format(
host=self._database_info.host,
port=self._database_info.bucketfs_port,
bucket=bucket,
password=self.bucketfs_write_password)
cmd = f"curl --silent --show-error --fail '{url}'"
return cmd
def upload_file(self, file_to_upload: str, upload_target: str):
self.logger.info("upload file %s to %s", file_to_upload, upload_target)
exit_code, log_output = self.run_command(
"upload", "ls " + str(Path(file_to_upload).parent))
command = self.generate_upload_command(file_to_upload, upload_target)
exit_code, log_output = self.run_command("upload", command)
if exit_code != 0:
self.write_logs(log_output)
raise Exception("Upload of %s failed, got following output %s" %
(file_to_upload, log_output))
return log_output
def generate_upload_command(self, file_to_upload, upload_target):
url = "http://*****:*****@{host}:{port}/{target}".format(
host=self._database_info.host,
port=self._database_info.bucketfs_port,
target=upload_target,
password=self.bucketfs_write_password)
cmd = f"curl --silent --show-error --fail -X PUT -T '{file_to_upload}' '{url}'"
return cmd
def run_command(self, command_type, cmd):
with self._get_docker_client() as docker_client:
test_container = docker_client.containers.get(
self._test_container_info.container_name)
self.logger.info("start %s command %s", command_type, cmd)
exit_code, output = test_container.exec_run(cmd=cmd)
self.logger.info("finish %s command %s", command_type, cmd)
log_output = cmd + "\n\n" + output.decode("utf-8")
return exit_code, log_output
def write_logs(self, output):
log_file = Path(self.get_log_path(), "log")
with log_file.open("w") as file:
file.write(output)
def get_log_file(self) -> str:
raise AbstractMethodException()
def get_pattern_to_wait_for(self) -> str:
raise AbstractMethodException()
def get_file_to_upload(self) -> str:
raise AbstractMethodException()
def get_upload_target(self) -> str:
raise AbstractMethodException()
def get_sync_time_estimation(self) -> int:
"""Estimated time in seconds which the bucketfs needs to extract and sync a uploaded file"""
raise AbstractMethodException()
class TestTask10(TestBaseTask):
parameter_1 = JsonPickleParameter(Data)
def run_task(self):
time.sleep(1)
print(self.parameter_1)
class SpawnTestContainer(DockerBaseTask):
environment_name = luigi.Parameter()
test_container_name = luigi.Parameter()
network_info = JsonPickleParameter(
DockerNetworkInfo, significant=False) # type: DockerNetworkInfo
ip_address_index_in_subnet = luigi.IntParameter(significant=False)
attempt = luigi.IntParameter(1)
reuse_test_container = luigi.BoolParameter(False, significant=False)
no_test_container_cleanup_after_success = luigi.BoolParameter(False, significant=False)
no_test_container_cleanup_after_failure = luigi.BoolParameter(False, significant=False)
docker_runtime = luigi.OptionalParameter(None, significant=False)
certificate_volume_name = luigi.OptionalParameter(None, significant=False)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.ip_address_index_in_subnet < 0:
raise Exception(
"ip_address_index_in_subnet needs to be greater than 0 got %s"
% self.ip_address_index_in_subnet)
def register_required(self):
self.test_container_image_future = \
self.register_dependency(self.create_child_task(task_class=DockerTestContainerBuild))
self.export_directory_future = \
self.register_dependency(self.create_child_task(task_class=CreateExportDirectory))
def is_reuse_possible(self) -> bool:
test_container_image_info = \
self.get_values_from_futures(self.test_container_image_future)["test-container"] # type: ImageInfo
test_container = None
with self._get_docker_client() as docker_client:
try:
test_container = docker_client.containers.get(self.test_container_name)
except Exception as e:
pass
ret_val = self.network_info.reused and self.reuse_test_container and \
test_container is not None and \
test_container_image_info.get_target_complete_name() in test_container.image.tags and \
test_container_image_info.image_state == ImageState.USED_LOCAL.name
return ret_val
def run_task(self):
subnet = netaddr.IPNetwork(self.network_info.subnet)
ip_address = str(subnet[2 + self.ip_address_index_in_subnet])
container_info = None
if self.is_reuse_possible():
container_info = self._try_to_reuse_test_container(ip_address, self.network_info)
if container_info is None:
container_info = self._create_test_container(ip_address, self.network_info)
with self._get_docker_client() as docker_client:
docker_client.containers.get(self.test_container_name)
self._copy_tests()
self.return_object(container_info)
def _copy_tests(self):
self.logger.warning("Copy tests in test container %s.", self.test_container_name)
with self._get_docker_client() as docker_client:
test_container = docker_client.containers.get(self.test_container_name)
try:
test_container.exec_run(cmd="rm -r /tests")
except:
pass
test_container.exec_run(cmd="cp -r /tests_src /tests")
def _try_to_reuse_test_container(self, ip_address: str,
network_info: DockerNetworkInfo) -> ContainerInfo:
self.logger.info("Try to reuse test container %s",
self.test_container_name)
container_info = None
try:
network_aliases = self._get_network_aliases()
container_info = self.create_container_info(ip_address, network_aliases, network_info)
except Exception as e:
self.logger.warning("Tried to reuse test container %s, but got Exeception %s. "
"Fallback to create new database.", self.test_container_name, e)
return container_info
def _create_test_container(self, ip_address,
network_info: DockerNetworkInfo) -> ContainerInfo:
self._remove_container(self.test_container_name)
self.logger.info(f"Creating new test container {self.test_container_name}")
test_container_image_info = \
self.get_values_from_futures(self.test_container_image_future)["test-container"]
# A later task which uses the test_container needs the exported container,
# but to access exported container from inside the test_container,
# we need to mount the release directory into the test_container.
exports_host_path = pathlib.Path(self._get_export_directory()).absolute()
tests_host_path = pathlib.Path("./tests").absolute()
volumes = {
exports_host_path: {
"bind": "/exports",
"mode": "rw"
},
tests_host_path: {
"bind": "/tests_src",
"mode": "rw"
},
}
if self.certificate_volume_name is not None:
volumes[self.certificate_volume_name] = {
"bind": "/certificates",
"mode": "ro"
}
with self._get_docker_client() as docker_client:
docker_unix_sockets = [i for i in docker_client.api.adapters.values()
if isinstance(i, unixconn.UnixHTTPAdapter)]
if len(docker_unix_sockets) > 0:
host_docker_socker_path = docker_unix_sockets[0].socket_path
volumes[host_docker_socker_path] = {
"bind": "/var/run/docker.sock",
"mode": "rw"
}
test_container = \
docker_client.containers.create(
image=test_container_image_info.get_target_complete_name(),
name=self.test_container_name,
network_mode=None,
command="sleep infinity",
detach=True,
volumes=volumes,
labels={"test_environment_name": self.environment_name, "container_type": "test_container"},
runtime=self.docker_runtime
)
docker_network = docker_client.networks.get(network_info.network_name)
network_aliases = self._get_network_aliases()
docker_network.connect(test_container, ipv4_address=ip_address, aliases=network_aliases)
test_container.start()
self.register_certificates(test_container)
container_info = self.create_container_info(ip_address, network_aliases, network_info)
return container_info
def _get_network_aliases(self):
network_aliases = ["test_container", self.test_container_name]
return network_aliases
def create_container_info(self, ip_address: str, network_aliases: List[str],
network_info: DockerNetworkInfo) -> ContainerInfo:
with self._get_docker_client() as docker_client:
test_container = docker_client.containers.get(self.test_container_name)
if test_container.status != "running":
raise Exception(f"Container {self.test_container_name} not running")
container_info = ContainerInfo(container_name=self.test_container_name,
ip_address=ip_address,
network_aliases=network_aliases,
network_info=network_info)
return container_info
def _get_export_directory(self):
return self.get_values_from_future(self.export_directory_future)
def _remove_container(self, container_name: str):
try:
with self._get_docker_client() as docker_client:
container = docker_client.containers.get(container_name)
container.remove(force=True)
self.logger.info(f"Removed container: name: '{container_name}', id: '{container.short_id}'")
except Exception as e:
pass
def register_certificates(self, test_container: Container):
if self.certificate_volume_name is not None:
script_name = "install_root_certificate.sh"
script_str = pkg_resources.resource_string(
PACKAGE_NAME,
f"test_container_config/{script_name}") # type: bytes
script_location_in_container = f"scripts/{script_name}"
copy_script_to_container(script_str.decode("UTF-8"), script_location_in_container, test_container)
exit_code, output = test_container.exec_run(f"bash {script_location_in_container}")
if exit_code != 0:
raise RuntimeError(f"Error installing certificates:{output.decode('utf-8')}")
def cleanup_task(self, success: bool):
if (success and not self.no_test_container_cleanup_after_success) or \
(not success and not self.no_test_container_cleanup_after_failure):
try:
self.logger.info(f"Cleaning up container %s", self.test_container_name)
self._remove_container(self.test_container_name)
except Exception as e:
self.logger.error(f"Error during removing container %s: %s", self.test_container_name, e)
class WaitForTestDockerDatabase(DockerBaseTask, DatabaseCredentialsParameter):
environment_name = luigi.Parameter()
test_container_info = JsonPickleParameter(
ContainerInfo, significant=False) # type: ContainerInfo
database_info = JsonPickleParameter(
DatabaseInfo, significant=False) # type: DatabaseInfo
db_startup_timeout_in_seconds = luigi.IntParameter(10 * 60,
significant=False)
attempt = luigi.IntParameter(1)
def run_task(self):
with self._get_docker_client() as docker_client:
test_container = docker_client.containers.get(
self.test_container_info.container_name)
db_container_name = self.database_info.container_info.container_name
db_container = docker_client.containers.get(db_container_name)
is_database_ready = \
self.wait_for_database_startup(test_container, db_container)
after_startup_db_log_file = self.get_log_path().joinpath(
"after_startup_db_log.tar.gz")
self.save_db_log_files_as_gzip_tar(after_startup_db_log_file,
db_container)
self.return_object(is_database_ready)
def wait_for_database_startup(self, test_container: Container,
db_container: Container):
container_log_thread, is_database_ready_thread = \
self.start_wait_threads(db_container, test_container)
is_database_ready = \
self.wait_for_threads(container_log_thread, is_database_ready_thread)
self.join_threads(container_log_thread, is_database_ready_thread)
return is_database_ready
def start_wait_threads(self, db_container, test_container):
startup_log_file = self.get_log_path().joinpath("startup.log")
container_log_thread = DBContainerLogThread(
db_container, self.logger, startup_log_file,
"Database Startup %s" % db_container.name)
container_log_thread.start()
is_database_ready_thread = IsDatabaseReadyThread(
self.logger, self.database_info, self.get_database_credentials(),
test_container)
is_database_ready_thread.start()
return container_log_thread, is_database_ready_thread
def join_threads(self, container_log_thread: DBContainerLogThread,
is_database_ready_thread: IsDatabaseReadyThread):
container_log_thread.stop()
is_database_ready_thread.stop()
container_log_thread.join()
is_database_ready_thread.join()
def wait_for_threads(self, container_log_thread: DBContainerLogThread,
is_database_ready_thread: IsDatabaseReadyThread):
is_database_ready = False
reason = None
start_time = datetime.now()
while (True):
if container_log_thread.error_message != None:
is_database_ready = False
reason = "error message in container log"
break
if is_database_ready_thread.finish:
is_database_ready = True
break
if self.timeout_occured(start_time):
reason = f"timeout after after {self.db_startup_timeout_in_seconds} seconds"
is_database_ready = False
break
time.sleep(1)
if not is_database_ready:
self.log_database_not_ready(container_log_thread,
is_database_ready_thread, reason)
container_log_thread.stop()
is_database_ready_thread.stop()
return is_database_ready
def log_database_not_ready(self,
container_log_thread: DBContainerLogThread,
is_database_ready_thread: IsDatabaseReadyThread,
reason):
container_log = '\n'.join(container_log_thread.complete_log)
log_information = f"""
========== IsDatabaseReadyThread output db connection: ============
{is_database_ready_thread.output_db_connection}
========== IsDatabaseReadyThread output bucketfs connection: ============
{is_database_ready_thread.output_bucketfs_connection}
========== Container-Log: ============
{container_log}
"""
self.logger.warning(
'Database startup failed for following reason "%s", here some debug information \n%s',
reason, log_information)
def timeout_occured(self, start_time):
timeout = timedelta(seconds=self.db_startup_timeout_in_seconds)
return datetime.now() - start_time > timeout
def save_db_log_files_as_gzip_tar(self, path: pathlib.Path,
database_container: Container):
stream, stat = database_container.get_archive("/exa/logs")
with gzip.open(path, "wb") as file:
for chunk in stream:
file.write(chunk)
def write_output(self, is_database_ready: bool):
with self.output().open("w") as file:
file.write(str(is_database_ready))