def __process_dockerfile(self, dockerfile):
"""
:param dockerfile_full_path:
:type dockerfile_full_path: str
:return: self
:rtype: self
"""
if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity():
self.line('<info>-> Processing: </info>%s' % dockerfile['image']['fullname'])
docker_image = dockerfile['image']['name']
parent_image_name = DockerfileUtility.image_basename(dockerfile['image']['from'])
parent_image_tag = DockerfileUtility.extract_image_name_tag(dockerfile['image']['from'])
if not parent_image_name in self.containers:
self.containers[parent_image_name] = 'scratch'
self.__append_tag(parent_image_name, 'latest')
self.containers[docker_image] = parent_image_name
self.__append_tag(docker_image, parent_image_tag)
return self
def pull_image(image):
print ' -> Pull base image %s ' % image
if configuration.get('dryRun'):
return True
pull_image_name = DockerfileUtility.image_basename(image)
pull_image_tag = DockerfileUtility.extract_image_name_tag(image)
pull_status = False
for retry_count in range(0, configuration.get('retry')):
pull_status = docker_client.pull_image(
name=pull_image_name,
tag=pull_image_tag,
)
if pull_status:
break
elif retry_count < (configuration.get('retry') - 1):
print ' failed, retrying... (try %s)' % (retry_count +
1)
else:
print ' failed, giving up'
if not pull_status:
return False
return True
def __process_dockerfile(self, dockerfile):
"""
:param dockerfile_full_path:
:type dockerfile_full_path: str
:return: self
:rtype: self
"""
if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity():
self.line('<info>-> Processing: </info>%s' %
dockerfile['image']['fullname'])
docker_image = dockerfile['image']['name']
parent_image_name = DockerfileUtility.image_basename(
dockerfile['image']['from'])
parent_image_tag = DockerfileUtility.extract_image_name_tag(
dockerfile['image']['from'])
if not parent_image_name in self.containers:
self.containers[parent_image_name] = 'scratch'
self.__append_tag(parent_image_name, 'latest')
self.containers[docker_image] = parent_image_name
self.__append_tag(docker_image, parent_image_tag)
return self
def pull_image(image):
print ' -> Pull base image %s ' % image
if configuration.get('dryRun'):
return True
pull_image_name = DockerfileUtility.image_basename(image)
pull_image_tag = DockerfileUtility.extract_image_name_tag(image)
pull_status = False
for retry_count in range(0, configuration.get('retry')):
pull_status = docker_client.pull_image(
name=pull_image_name,
tag=pull_image_tag,
)
if pull_status:
break
elif retry_count < (configuration.get('retry') - 1):
print ' failed, retrying... (try %s)' % (retry_count + 1)
else:
print ' failed, giving up'
if not pull_status:
return False
return True
def task_dependency_puller(docker_client, dockerfileList, configuration,
task):
"""
Pulls dependency images before building
"""
def pull_image(image):
print ' -> Pull base image %s ' % image
if configuration.get('dryRun'):
return True
pull_image_name = DockerfileUtility.image_basename(image)
pull_image_tag = DockerfileUtility.extract_image_name_tag(image)
pull_status = False
for retry_count in range(0, configuration.get('retry')):
pull_status = docker_client.pull_image(
name=pull_image_name,
tag=pull_image_tag,
)
if pull_status:
break
elif retry_count < (configuration.get('retry') - 1):
print ' failed, retrying... (try %s)' % (retry_count +
1)
else:
print ' failed, giving up'
if not pull_status:
return False
return True
image_list = []
for dockerfile in dockerfileList:
# Pull base image (FROM: xxx) first
if DockerfileUtility.check_if_base_image_needs_pull(
dockerfile['image']['from'], configuration):
image_list.append(dockerfile['image']['from'])
# Pull straged images (multi-stage dockerfiles)
for multiStageImage in dockerfile['image']['multiStageImages']:
if DockerfileUtility.check_if_base_image_needs_pull(
multiStageImage, configuration):
image_list.append(multiStageImage)
# filter only unique image names
image_list = set(image_list)
# pull images
for image in set(image_list):
if not pull_image(image):
return False
return True
def task_run(docker_client, dockerfile, configuration, task):
"""
Build one Dockerfile
"""
pull_parent_image = DockerfileUtility.check_if_base_image_needs_pull(dockerfile, configuration)
if configuration.get('dryRun'):
print ' from: %s (pull: %s)' % (dockerfile['image']['from'], ('yes' if pull_parent_image else 'no'))
print ' path: %s' % dockerfile['path']
print ' dep: %s' % (DockerBuildTaskLoader.human_task_name_list(task.task_dep) if task.task_dep else 'none')
return True
# Pull base image (FROM: xxx) first
if pull_parent_image:
print ' -> Pull base image %s ' % dockerfile['image']['from']
pull_image_name = DockerfileUtility.image_basename(dockerfile['image']['from'])
pull_image_tag = DockerfileUtility.extract_image_name_tag(dockerfile['image']['from'])
pull_status = False
for retry_count in range(0, configuration.get('retry')):
pull_status = docker_client.pull_image(
name=pull_image_name,
tag=pull_image_tag,
)
if pull_status:
break
elif retry_count < (configuration.get('retry') - 1):
print ' failed, retrying... (try %s)' % (retry_count+1)
else:
print ' failed, giving up'
if not pull_status:
return False
## Build image
print ' -> Building image %s ' % dockerfile['image']['fullname']
build_status = False
for retry_count in range(0, configuration.get('retry')):
build_status = docker_client.build_dockerfile(
path=dockerfile['path'],
name=dockerfile['image']['fullname'],
nocache=configuration.get('dockerBuild.noCache'),
)
if build_status:
break
elif retry_count < (configuration.get('retry')-1):
print ' failed, retrying... (try %s)' % (retry_count+1)
else:
print ' failed, giving up'
return build_status
def task_dependency_puller(docker_client, dockerfileList, configuration, task):
"""
Pulls dependency images before building
"""
def pull_image(image):
print ' -> Pull base image %s ' % image
if configuration.get('dryRun'):
return True
pull_image_name = DockerfileUtility.image_basename(image)
pull_image_tag = DockerfileUtility.extract_image_name_tag(image)
pull_status = False
for retry_count in range(0, configuration.get('retry')):
pull_status = docker_client.pull_image(
name=pull_image_name,
tag=pull_image_tag,
)
if pull_status:
break
elif retry_count < (configuration.get('retry') - 1):
print ' failed, retrying... (try %s)' % (retry_count + 1)
else:
print ' failed, giving up'
if not pull_status:
return False
return True
image_list = []
for dockerfile in dockerfileList:
# Pull base image (FROM: xxx) first
if DockerfileUtility.check_if_base_image_needs_pull(dockerfile['image']['from'], configuration):
image_list.append(dockerfile['image']['from'])
# Pull straged images (multi-stage dockerfiles)
for multiStageImage in dockerfile['image']['multiStageImages']:
if DockerfileUtility.check_if_base_image_needs_pull(multiStageImage, configuration):
image_list.append(multiStageImage)
# filter only unique image names
image_list = set(image_list)
# pull images
for image in set(image_list):
if not pull_image(image):
return False
return True
def load_tasks(self, cmd, opt_values, pos_args):
"""
DOIT task list generator
"""
config = {'verbosity': self.configuration.get('verbosity')}
dockerfile_list = DockerfileUtility.find_dockerfiles_in_path(
base_path=self.configuration.get('dockerPath'),
path_regex=self.configuration.get('docker.pathRegex'),
image_prefix=self.configuration.get('docker.imagePrefix'),
whitelist=self.configuration.get('whitelist'),
blacklist=self.configuration.get('blacklist'),
)
dockerfile_list = self.process_dockerfile_list(dockerfile_list)
#import json,sys;print json.dumps(dockerfile_list, sort_keys=True, indent = 4, separators = (',', ': '));sys.exit(0);
tasklist = self.generate_task_list(dockerfile_list)
if not tasklist or len(tasklist) == 0:
raise Exception('No tasks found')
tasklist = self.process_tasklist(tasklist)
return tasklist, config
def run_task(self, configuration):
template_path = configuration.get('templatePath')
dockerfile_path = configuration.get('dockerPath')
whitelist = self.get_whitelist()
blacklist = self.get_blacklist()
if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity():
self.line('<info>-> </info><comment>docker path</comment> : %s' % dockerfile_path)
self.line('<info>-> </info><comment>template path </comment> : %s' % template_path)
if whitelist:
self.line('<info>-> </info><comment>whitelist </comment> :')
for crit in whitelist:
self.line("\t * %s" % crit)
if blacklist:
self.line('<info>-> </info><comment>blacklist </comment> :')
for crit in blacklist:
self.line("\t * %s" % crit)
self.template = Environment(
autoescape=False,
loader=FileSystemLoader([template_path]),
trim_blocks=False
)
for file in DockerfileUtility.find_file_in_path(dockerfile_path=dockerfile_path, filename="Dockerfile.jinja2", whitelist=whitelist, blacklist=blacklist):
self.process_dockerfile(file)
def process_dockerfile_list(self, dockerfile_list):
"""
Prepare dockerfile list with dependency and also add "auto latest tag" images
"""
image_list = [x['image']['fullname'] for x in dockerfile_list if x['image']['fullname']]
autoLatestTagImageList = []
for dockerfile in dockerfile_list:
# Calculate dependency
dockerfile['dependency'] = False
if dockerfile['image']['from'] and dockerfile['image']['from'] in image_list:
dockerfile['dependency'] = dockerfile['image']['from']
# Process auto latest tag
if self.configuration.get('docker.autoLatestTag') and dockerfile['image']['tag'] == self.configuration.get('docker.autoLatestTag'):
imageNameLatest = DockerfileUtility.generate_image_name_with_tag_latest(dockerfile['image']['fullname'])
if imageNameLatest not in image_list:
autoLatestTagImage = copy.deepcopy(dockerfile)
autoLatestTagImage['image']['fullname'] = imageNameLatest
autoLatestTagImage['image']['tag'] = 'latest'
autoLatestTagImage['dependency'] = dockerfile['image']['fullname']
autoLatestTagImageList.append(autoLatestTagImage)
# Add auto latest tag images to dockerfile list
dockerfile_list.extend(autoLatestTagImageList)
return dockerfile_list
def load_tasks(self, cmd, opt_values, pos_args):
"""
DOIT task list generator
"""
config = {'verbosity': self.configuration.get('verbosity')}
dockerfile_list = DockerfileUtility.find_dockerfiles_in_path(
base_path=self.configuration.get('dockerPath'),
path_regex=self.configuration.get('docker.pathRegex'),
image_prefix=self.configuration.get('docker.imagePrefix'),
whitelist=self.configuration.get('whitelist'),
blacklist=self.configuration.get('blacklist'),
)
dockerfile_list = self.process_dockerfile_list(dockerfile_list)
#import json,sys;print json.dumps(dockerfile_list, sort_keys=True, indent = 4, separators = (',', ': '));sys.exit(0);
tasklist = self.generate_task_list(dockerfile_list)
if not tasklist or len(tasklist) == 0:
raise Exception('No tasks found')
tasklist = self.process_tasklist(tasklist)
return tasklist, config
def run_task(self, configuration):
template_path = configuration.get('templatePath')
dockerfile_path = configuration.get('dockerPath')
whitelist = self.get_whitelist()
blacklist = self.get_blacklist()
if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity():
self.line('<info>-> </info><comment>docker path</comment> : %s' %
dockerfile_path)
self.line(
'<info>-> </info><comment>template path </comment> : %s' %
template_path)
if whitelist:
self.line('<info>-> </info><comment>whitelist </comment> :')
for crit in whitelist:
self.line("\t * %s" % crit)
if blacklist:
self.line('<info>-> </info><comment>blacklist </comment> :')
for crit in blacklist:
self.line("\t * %s" % crit)
self.template = Environment(autoescape=False,
loader=FileSystemLoader([template_path]),
trim_blocks=False)
for file in DockerfileUtility.find_file_in_path(
dockerfile_path=dockerfile_path,
filename="Dockerfile.jinja2",
whitelist=whitelist,
blacklist=blacklist):
self.process_dockerfile(file)
def run_task(self, configuration):
if self.option('output'):
configuration.set('imagePath', self.option('output'))
if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity():
self.line('<info>ALL :</info> %s' % self.option('all'))
self.line('<info>output :</info> %s' %
configuration.get('imagePath'))
self.line('<info>format :</info> %s' % self.option('format'))
self.line('<info>dockerPath :</info> %s' %
configuration.get('dockerPath'))
self.line('<info>filename :</info> %s' % self.option('filename'))
self.__load_configuration()
dockerfileList = DockerfileUtility.find_dockerfiles_in_path(
base_path=configuration.get('dockerPath'),
path_regex=configuration.get('docker.pathRegex'),
image_prefix=configuration.get('docker.imagePrefix'),
)
for dockerfile in dockerfileList:
self.__process_dockerfile(dockerfile)
dia = self.build_graph()
dia.render()
def run_task(self, configuration):
dockerfile_list = DockerfileUtility.find_dockerfiles_in_path(
base_path=configuration.get('dockerPath'),
path_regex=configuration.get('docker.pathRegex'),
image_prefix=configuration.get('docker.imagePrefix'),
whitelist=configuration.get('whitelist'),
blacklist=configuration.get('blacklist'),
)
image_fail_list = []
docker_command = self.argument('docker command')
for dockerfile in dockerfile_list:
title = dockerfile['image']['fullname']
print title
print '~' * len(title)
if configuration['dryRun']:
print ' exec: %s' % (docker_command)
else:
cmd = [
'docker',
'run',
'-t',
'--rm',
dockerfile['image']['fullname'],
'bash',
'-c',
'%s' % (' '.join(docker_command))
]
status = Command.execute(cmd)
if status:
print colored(' -> successfull', 'green')
else:
print colored(' -> failed', 'red')
image_fail_list.append(dockerfile['image']['fullname'])
print ''
print ''
if len(image_fail_list) >= 1:
print ''
print colored(' => failed images (%s):' % (str(len(image_fail_list))), 'red')
for image in image_fail_list:
print ' - %s ' % image
print ''
return False
else:
return True
def run_task(self, configuration):
dockerfile_list = DockerfileUtility.find_dockerfiles_in_path(
base_path=configuration.get('dockerPath'),
path_regex=configuration.get('docker.pathRegex'),
image_prefix=configuration.get('docker.imagePrefix'),
whitelist=configuration.get('whitelist'),
blacklist=configuration.get('blacklist'),
)
image_fail_list = []
docker_command = self.argument('docker command')
for dockerfile in dockerfile_list:
title = dockerfile['image']['fullname']
print title
print '~' * len(title)
if configuration['dryRun']:
print ' exec: %s' % (docker_command)
else:
cmd = [
'docker', 'run', '-t', '--rm',
dockerfile['image']['fullname'], 'bash', '-c',
'%s' % (' '.join(docker_command))
]
status = Command.execute(cmd)
if status:
print colored(' -> successfull', 'green')
else:
print colored(' -> failed', 'red')
image_fail_list.append(dockerfile['image']['fullname'])
print ''
print ''
if len(image_fail_list) >= 1:
print ''
print colored(
' => failed images (%s):' % (str(len(image_fail_list))), 'red')
for image in image_fail_list:
print ' - %s ' % image
print ''
return False
else:
return True
def init_docker_image_list(self, docker_image=False):
"""
Init and build list of available docker images
"""
if docker_image:
self.docker_image_list.append(docker_image)
else:
dockerfile_list = DockerfileUtility.find_dockerfiles_in_path(
base_path=self.configuration.get('dockerPath'),
path_regex=self.configuration.get('docker.pathRegex'),
image_prefix=self.configuration.get('docker.imagePrefix'),
whitelist=self.configuration.get('whitelist'),
blacklist=self.configuration.get('blacklist'),
)
for image in dockerfile_list:
self.docker_image_list.append(image['image']['fullname'])
def init_docker_image_list(self, docker_image=False):
"""
Init and build list of available docker images
"""
if docker_image:
self.docker_image_list.append(docker_image)
else:
dockerfile_list = DockerfileUtility.find_dockerfiles_in_path(
base_path=self.configuration.get('dockerPath'),
path_regex=self.configuration.get('docker.pathRegex'),
image_prefix=self.configuration.get('docker.imagePrefix'),
whitelist=self.configuration.get('whitelist'),
blacklist=self.configuration.get('blacklist'),
)
for image in dockerfile_list:
self.docker_image_list.append(image['image']['fullname'])
def process_dockerfile_list(self, dockerfile_list):
"""
Prepare dockerfile list with dependency and also add "auto latest tag" images
"""
# Process auto latest tag
autoLatestTagImageList = []
image_list = [x['image']['fullname'] for x in dockerfile_list if x['image']['fullname']]
for dockerfile in dockerfile_list:
if self.configuration.get('docker.autoLatestTag') and dockerfile['image']['tag'] == self.configuration.get('docker.autoLatestTag'):
imageNameLatest = DockerfileUtility.generate_image_name_with_tag_latest(dockerfile['image']['fullname'])
if imageNameLatest not in image_list:
autoLatestTagImage = copy.deepcopy(dockerfile)
autoLatestTagImage['image']['fullname'] = imageNameLatest
autoLatestTagImage['image']['tag'] = 'latest'
autoLatestTagImage['image']['duplicate'] = True
if not 'dependency' in autoLatestTagImage:
autoLatestTagImage['dependency'] = []
autoLatestTagImage['dependency'].append(dockerfile['image']['fullname'])
autoLatestTagImageList.append(autoLatestTagImage)
# Add auto latest tag images to dockerfile list
dockerfile_list.extend(autoLatestTagImageList)
# Calculate dependency
image_list = [x['image']['fullname'] for x in dockerfile_list if x['image']['fullname']]
for dockerfile in dockerfile_list:
if not 'dependency' in dockerfile:
dockerfile['dependency'] = []
# add image from if it is a dependency
if dockerfile['image']['from'] and dockerfile['image']['from'] in image_list:
dockerfile['dependency'].append(dockerfile['image']['from'])
# add multi stage image if it is a dependency
for multiStageImage in dockerfile['image']['multiStageImages']:
if multiStageImage in image_list:
dockerfile['dependency'].append(multiStageImage)
# unique list
unique_dep_list = []
[unique_dep_list.append(item) for item in dockerfile['dependency'] if item not in unique_dep_list]
dockerfile['dependency'] = unique_dep_list
return dockerfile_list
def process_dockerfile_list(self, dockerfile_list):
"""
Prepare dockerfile list with dependency and also add "auto latest tag" images
"""
# Process auto latest tag
autoLatestTagImageList = []
image_list = [
x['image']['fullname'] for x in dockerfile_list
if x['image']['fullname']
]
for dockerfile in dockerfile_list:
if self.configuration.get('docker.autoLatestTag') and dockerfile[
'image']['tag'] == self.configuration.get(
'docker.autoLatestTag'):
imageNameLatest = DockerfileUtility.generate_image_name_with_tag_latest(
dockerfile['image']['fullname'])
if imageNameLatest not in image_list:
autoLatestTagImage = copy.deepcopy(dockerfile)
autoLatestTagImage['image']['fullname'] = imageNameLatest
autoLatestTagImage['image']['tag'] = 'latest'
autoLatestTagImage['dependency'] = dockerfile['image'][
'fullname']
autoLatestTagImageList.append(autoLatestTagImage)
# Add auto latest tag images to dockerfile list
dockerfile_list.extend(autoLatestTagImageList)
# Calculate dependency
image_list = [
x['image']['fullname'] for x in dockerfile_list
if x['image']['fullname']
]
for dockerfile in dockerfile_list:
if not 'dependency' in dockerfile:
dockerfile['dependency'] = False
if dockerfile['image']['from'] and dockerfile['image'][
'from'] in image_list:
dockerfile['dependency'] = dockerfile['image']['from']
return dockerfile_list
def run_task(self, configuration):
if self.option('output'):
configuration.set('imagePath', self.option('output'))
if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity():
self.line('<info>ALL :</info> %s' % self.option('all'))
self.line('<info>output :</info> %s' % configuration.get('imagePath'))
self.line('<info>format :</info> %s' % self.option('format'))
self.line('<info>dockerPath :</info> %s' % configuration.get('dockerPath'))
self.line('<info>filename :</info> %s' % self.option('filename'))
self.__load_configuration()
dockerfileList = DockerfileUtility.find_dockerfiles_in_path(
base_path=configuration.get('dockerPath'),
path_regex=configuration.get('docker.pathRegex'),
image_prefix=configuration.get('docker.imagePrefix'),
)
for dockerfile in dockerfileList:
self.__process_dockerfile(dockerfile)
dia = self.build_graph()
dia.render()
def task_run(docker_client, dockerfile, configuration, task):
"""
Build one Dockerfile
"""
pull_parent_image = DockerfileUtility.check_if_base_image_needs_pull(
dockerfile, configuration)
if configuration.get('dryRun'):
print ' from: %s (pull: %s)' % (dockerfile['image']['from'],
('yes' if pull_parent_image
else 'no'))
print ' path: %s' % dockerfile['path']
print ' dep: %s' % (
DockerBuildTaskLoader.human_task_name_list(task.task_dep)
if task.task_dep else 'none')
return True
# Pull base image (FROM: xxx) first
if pull_parent_image:
print ' -> Pull base image %s ' % dockerfile['image']['from']
pull_image_name = DockerfileUtility.image_basename(
dockerfile['image']['from'])
pull_image_tag = DockerfileUtility.extract_image_name_tag(
dockerfile['image']['from'])
pull_status = False
for retry_count in range(0, configuration.get('retry')):
pull_status = docker_client.pull_image(
name=pull_image_name,
tag=pull_image_tag,
)
if pull_status:
break
elif retry_count < (configuration.get('retry') - 1):
print ' failed, retrying... (try %s)' % (retry_count +
1)
else:
print ' failed, giving up'
if not pull_status:
return False
## Build image
print ' -> Building image %s ' % dockerfile['image']['fullname']
build_status = False
for retry_count in range(0, configuration.get('retry')):
build_status = docker_client.build_dockerfile(
path=dockerfile['path'],
name=dockerfile['image']['fullname'],
nocache=configuration.get('dockerBuild.noCache'),
)
if build_status:
break
elif retry_count < (configuration.get('retry') - 1):
print ' failed, retrying... (try %s)' % (retry_count + 1)
else:
print ' failed, giving up'
return build_status
