def __init__(self,
image_name,
path_images=None,
image_tag='latest') -> object:
self.api_docker = API_Docker()
self.ecr = ECR()
self.aws_config = AWS_Config()
self.image_name = image_name
self.image_tag = image_tag
self.path_images = path_images or path_combine(__file__,
'../../images')
class Create_Image_ECR:
def __init__(self, image_name, image_tag='latest'):
self.api_docker = API_Docker()
self.ecr = ECR()
self.aws_config = AWS_Config()
self.image_name = image_name
self.image_tag = image_tag
def build_image(self):
repository = self.image_repository()
tag = self.image_tag
result = self.api_docker.image_build(path=self.path_image(),
repository=repository,
tag=tag)
return result.get('status') == 'ok'
def create_repository(self):
self.ecr.repository_create(self.image_name)
return self.ecr.repository_exists(self.image_name)
def image_repository(self):
account_id = self.aws_config.aws_session_account_id()
region = self.aws_config.aws_session_region_name()
return f'{account_id}.dkr.ecr.{region}.amazonaws.com/{self.image_name}'
def ecr_login(self):
auth_data = self.ecr.authorization_token()
return self.api_docker.registry_login(
registry=auth_data.get('registry'),
username=auth_data.get('username'),
password=auth_data.get('password'))
def path_image(self):
return path_combine(self.path_images(), self.image_name)
def path_images(self):
return path_combine(__file__, '../../images')
def push_image(self):
return self.api_docker.image_push(self.image_repository(),
self.image_tag)
def run(self):
create_repository = self.create_repository()
ecr_login = self.ecr_login()
build_image = self.build_image()
push_image = self.push_image()
return {
'create_repository': create_repository,
'ecr_login': ecr_login,
'build_image': build_image,
'push_image': push_image
}
def __init__(self):
self.image_repository = "k8_kubectl__icap_client"
self.image_tag = "latest"
self.image_name = f"{self.image_repository}:{self.image_tag}"
self.icap_client_path = '/usr/local/c-icap/bin/c-icap-client'
self.api_docker = API_Docker()
class Icap_Client:
def __init__(self):
self.image_repository = "k8_kubectl__icap_client"
self.image_tag = "latest"
self.image_name = f"{self.image_repository}:{self.image_tag}"
self.icap_client_path = '/usr/local/c-icap/bin/c-icap-client'
self.api_docker = API_Docker()
def extract_icap_headers(self, output):
headers = {}
icap_headers = {
"icap_headers": headers,
"icap_server": None,
"icap_port": None,
"options_headers": [],
"docker_ip": None,
}
regex_target = "ICAP server:(.*), ip:(.*), port:(.*)"
regex_status = "(.*)\/1.0 ([0-9]*) OK"
regex_service = "Via: (.*)\/1.0 (.*) \((.*)\)"
in_icap_headers = False
for line in str_lines(output):
line = line.strip()
if line.startswith('ICAP server'):
match = search(regex_target, line)
icap_headers['icap_server'] = match.group(1)
icap_headers['docker_ip'] = match.group(2)
icap_headers['icap_port'] = match.group(3)
if line.startswith('Add resp header'):
(key, value) = line.split(':', 1)
icap_headers['options_headers'].append(value)
if line == 'ICAP HEADERS:':
in_icap_headers = True
continue
if in_icap_headers:
match_status = search(regex_status, line)
match_service = search(regex_service, line)
if match_status:
icap_headers['response_status'] = line
icap_headers['schema'] = match_status.group(1)
icap_headers['status_code'] = match_status.group(2)
elif match_service:
icap_headers['response_status'] = line
icap_headers['schema'] = match_service.group(1)
icap_headers['k8_container'] = match_service.group(2)
icap_headers['icap_service'] = match_service.group(3)
else:
if line.find(':') > -1:
(key, value) = line.split(':', 1)
icap_headers[key] = value.strip()
return icap_headers
def extract_time(self, output):
regex_time_data = '\nreal\t(.*)\nuser\t(.*)\nsys\t(.*)\n'
regex_time = '(.*)m(.*)\.(.*)s'
match_time_data = search(regex_time_data, output)
time_str = ''
time_date = ''
if match_time_data:
time_str = match_time_data.group(1)
match_time = search(regex_time, time_str)
minutes = to_int(match_time.group(1))
seconds = to_int(match_time.group(2))
micro_seconds = to_int(match_time.group(3)) * 1000
time_date = time(0, minutes, seconds, micro_seconds)
#output = remove(output, match_time_data.group(0))
return {"time_str": time_str, 'time_date': time_date}
def icap_echo(self, icap_server):
icap_params = f'-i {icap_server} ' #-s gw_rebuild
return self.icap_run(icap_params)
def icap_echo_service(self, icap_server, service_name):
icap_params = f'-i {icap_server} -s {service_name}'
return self.icap_run(icap_params)
def icap_help(self):
return self.icap_run('-h').get('docker_run').get('stdout')
def get_processing_local_config(self, target_file_name):
icap_processing_folder = path_combine(current_temp_folder(),
'icap_processing_folder')
icap_session = new_guid()
icap_session_folder = path_combine(icap_processing_folder,
icap_session)
input_folder = create_folder_in_parent(icap_session_folder, 'input')
output_folder = create_folder_in_parent(icap_session_folder, 'output')
input_file = path_combine(input_folder, target_file_name)
output_file = path_combine(output_folder, target_file_name)
return {
"temp_folder": icap_session_folder,
"session_id": icap_session,
"input_file": input_file,
"output_file": output_file
}
def get_processing_docker_config(self, target_file_name):
icap_folder = '/icap_folder'
input_folder = path_combine(icap_folder, 'input')
output_folder = path_combine(icap_folder, 'output')
input_file = path_combine(input_folder, target_file_name)
output_file = path_combine(output_folder, target_file_name)
return {
"icap_folder": icap_folder,
"input_file": input_file,
"output_file": output_file
}
def get_processing_config(self, target_file):
target_file_name = file_name(target_file)
config = {
"target_file": target_file,
"file_name": target_file_name,
"local_config": self.get_processing_local_config(target_file_name),
"docker_config":
self.get_processing_docker_config(target_file_name)
}
file_copy(source=target_file,
destination=config.get('local_config').get('input_file'))
return config
def get_processing_result(self, config, icap_result):
target_file = config.get('local_config').get('input_file')
rebuilt_file = config.get('local_config').get('output_file')
processing_result = {
"config": config,
"icap_result": icap_result,
'file_sizes': {
"original": file_size(target_file),
"rebuilt": None
},
"md5s": {
"target": file_md5(target_file),
"rebuilt": None
}
}
if file_exists(rebuilt_file):
processing_result.get("md5s")['rebuilt'] = file_md5(rebuilt_file)
processing_result.get("file_sizes")['rebuilt'] = file_size(
rebuilt_file)
return processing_result
def icap_process_file(self, target_ip, target_service, target_file):
config = self.get_processing_config(target_file)
path_local_icap_data = config.get('local_config').get('temp_folder')
path_docker_icap_data = config.get('docker_config').get('icap_folder')
input_file = config.get('docker_config').get('input_file')
output_file = config.get('docker_config').get('output_file')
docker_options = {
'key': '-v',
'value': f'{path_local_icap_data}:{path_docker_icap_data}'
}
command = f'-v -i {target_ip} -s {target_service} -f {input_file} -o {output_file} -d 10'
#command = f'-i {target_ip}'
icap_result = self.icap_run(command, options=docker_options)
processing_result = self.get_processing_result(config, icap_result)
return processing_result
def icap_run(self, params=None, options=None):
icap_params = f'time {self.icap_client_path} {params}'
docker_run = self.api_docker.docker_run_bash(self.image_name,
image_params=icap_params,
options=options)
console = docker_run.get('stdout') + docker_run.get('stderr')
duration = self.extract_time(console)
icap_headers = self.extract_icap_headers(console)
return {
"icap_params": icap_params,
'docker_run': docker_run,
'duration': duration,
'icap_headers': icap_headers
}
def icap_run_command(self, command, options=None):
output = self.api_docker.docker_run_bash(self.image_name,
image_params=command,
options=options)
console = output.get('stdout') + output.get('stderr')
return console
def image_exists(self):
return self.image_name in self.api_docker.images_names()
def image_build(self):
path = self.path_folder_with_docker_file()
result = self.api_docker.image_build(path, self.image_repository,
self.image_tag)
return result.get('status') == 'ok'
def path_folder_with_docker_file(self):
return path_combine(__file__, '../../../docker/icap-client')
def set_icap_timeout(self, value):
self.api_docker.set_docker_run_timeout(value)
class Create_Image_ECR:
def __init__(self,
image_name,
path_images=None,
image_tag='latest') -> object:
self.api_docker = API_Docker()
self.ecr = ECR()
self.aws_config = AWS_Config()
self.image_name = image_name
self.image_tag = image_tag
self.path_images = path_images or path_combine(__file__,
'../../images')
def build_image(self):
repository = self.image_repository()
tag = self.image_tag
result = self.api_docker.image_build(path=self.path_image(),
repository=repository,
tag=tag)
return result.get('status') == 'ok'
def create_repository(self):
self.ecr.repository_create(self.image_name)
return self.ecr.repository_exists(self.image_name)
def image_name(self):
return f'{self.image_repository()}:{self.image_tag}'
def image_repository(self):
account_id = self.aws_config.aws_session_account_id()
region = self.aws_config.aws_session_region_name()
return f'{account_id}.dkr.ecr.{region}.amazonaws.com/{self.image_name}'
def ecr_login(self):
auth_data = self.ecr.authorization_token()
return self.api_docker.registry_login(
registry=auth_data.get('registry'),
username=auth_data.get('username'),
password=auth_data.get('password'))
def path_image(self):
return path_combine(self.path_images, self.image_name)
def push_image(self):
json_lines = self.api_docker.image_push(self.image_repository(),
self.image_tag)
return json_lines
def run_locally(self):
image_name = self.image_name()
return self.api_docker.docker_run(image_name)
def create(self):
create_repository = self.create_repository()
ecr_login = self.ecr_login()
build_image = self.build_image()
push_image = self.push_image()
# status = create_repository and \
# build_image and \
# ecr_login.get('Status') == 'Login Succeeded' #\
# # push_image ????\ # todo: add success/error detector to push_image images logs (use json_lines_parse to parse string into json)
return {
'create_repository': create_repository,
'ecr_login': ecr_login,
'build_image': build_image,
'push_image': push_image,
#'status' : status
}
def setUp(self) -> None:
self.api_docker = API_Docker()
self.path_docker_images = path_combine(
__file__, '../../_test_data/docker_images')
print()
class test_API_Docker(TestCase):
def setUp(self) -> None:
self.api_docker = API_Docker()
self.path_docker_images = path_combine(
__file__, '../../_test_data/docker_images')
print()
def test_client(self):
assert type(self.api_docker.client()).__name__ == 'DockerClient'
def test_container_run(self):
assert 'Hello from Docker!' in self.api_docker.container_run(
'hello-world').get('output')
assert self.api_docker.container_run(
'hello-world', 'bbbb'
).get('error') == (
'404 Client Error for '
'http+docker://localhost/v1.40/images/create?tag=bbbb&fromImage=hello-world: '
'Not Found ("manifest for hello-world:bbbb not found: manifest unknown: '
'manifest unknown")')
assert self.api_docker.container_run('aaaa', 'bbbb').get('error') == (
'404 Client Error for '
'http+docker://localhost/v1.40/images/create?tag=bbbb&fromImage=aaaa: '
'Not Found ("pull access denied for aaaa, repository does not '
"exist or may require 'docker login': denied: requested access to the "
'resource is denied")')
def test_containers(self):
assert type(
self.api_docker.containers()
) is list # todo once we create a container per execution change this to reflect that
def test_docker_params_append_options(self):
docker_params = ['run']
options = {'key': '-v', 'value': '/a:/b'}
result = self.api_docker.docker_params_append_options(
docker_params=docker_params, options=options)
assert result == ['run', '-v', '/a:/b']
options = [{
'key': '-v',
'value': '/c:/d'
}, {
'key': '-v',
'value': '/e:/f'
}]
result = self.api_docker.docker_params_append_options(
docker_params, options)
assert result == ['run', '-v', '/a:/b', '-v', '/c:/d', '-v', '/e:/f']
def test_image_build(self):
target_image = 'centos'
expected_size = 209348126
folder_dockerFile = path_combine(self.path_docker_images, target_image)
path_dockerfile = path_combine(folder_dockerFile, 'Dockerfile')
repository = "osbot_docker__test_image_build"
tag = "abc"
image_name = f"{repository}:{tag}"
assert folder_exists(folder_dockerFile)
assert file_exists(path_dockerfile)
result = self.api_docker.image_build(folder_dockerFile, repository,
tag)
build_logs = result.get('build_logs')
image = result.get('image')
status = result.get('status')
tags = result.get('tags')
assert self.api_docker.image_exists(repository, tag)
assert status == 'ok'
assert image_name in tags
assert image_name in self.api_docker.images_names()
assert image.get('Size') == expected_size
assert next(build_logs) == {'stream': 'Step 1/3 : FROM centos:8'}
assert self.api_docker.image_delete(repository, tag) is True
assert image_name not in self.api_docker.images_names()
def test_image_build__bad_data(self):
assert self.api_docker.image_build(None, None).get(
'error') == 'Either path or fileobj needs to be provided.'
assert self.api_docker.image_build('', None).get(
'error') == 'You must specify a directory to build in path'
# todo: find out why in GH Actions the line below throws the error: AttributeError: 'APIError' object has no attribute 'msg'
#assert self.api_docker.image_build(temp_folder(), None).get('exception').msg.get('message') == 'Cannot locate specified Dockerfile: Dockerfile'
def test_image_build_scratch(self):
path = path_combine(self.path_docker_images, 'scratch')
repository = 'scratch'
tag = 'latest'
result = self.api_docker.image_build(path=path,
repository=repository,
tag=tag)
assert result.get('image').get('Size') == 0
def test_image_info(self):
assert self.api_docker.image_info(random_string()) is None
def test_image_exists(self):
assert self.api_docker.image_exists(random_string()) is False
def test_image_pull(self):
repository = 'centos'
tag = '8'
image = self.api_docker.image_pull(repository, tag)
assert image.tags == ['centos:8']
assert self.api_docker.container_run(repository, tag, "pwd") == {
'output': '/',
'status': 'ok'
}
assert self.api_docker.container_run(
repository, tag, "cat /etc/redhat-release") == {
'output': 'CentOS Linux release 8.3.2011',
'status': 'ok'
}
def test_images(self):
images = self.api_docker.images()
assert len(images) > 0
def test_images_names(self):
names = self.api_docker.images_names()
assert 'hello-world:latest' in names
def test_server_info(self):
server_info = self.api_docker.server_info()
assert 'KernelMemory' in server_info
assert lower(server_info.get('OSType')) == 'linux'
def __init__(self, image_name, image_tag='latest'):
self.api_docker = API_Docker()
self.ecr = ECR()
self.aws_config = AWS_Config()
self.image_name = image_name
self.image_tag = image_tag