def __process_dockerfile(self, dockerfile): """ :param dockerfile_full_path: :type dockerfile_full_path: str :return: self :rtype: self """ if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): self.line('<info>-> Processing: </info>%s' % dockerfile['image']['fullname']) docker_image = dockerfile['image']['name'] parent_image_name = DockerfileUtility.image_basename(dockerfile['image']['from']) parent_image_tag = DockerfileUtility.extract_image_name_tag(dockerfile['image']['from']) if not parent_image_name in self.containers: self.containers[parent_image_name] = 'scratch' self.__append_tag(parent_image_name, 'latest') self.containers[docker_image] = parent_image_name self.__append_tag(docker_image, parent_image_tag) return self
def pull_image(image): print ' -> Pull base image %s ' % image if configuration.get('dryRun'): return True pull_image_name = DockerfileUtility.image_basename(image) pull_image_tag = DockerfileUtility.extract_image_name_tag(image) pull_status = False for retry_count in range(0, configuration.get('retry')): pull_status = docker_client.pull_image( name=pull_image_name, tag=pull_image_tag, ) if pull_status: break elif retry_count < (configuration.get('retry') - 1): print ' failed, retrying... (try %s)' % (retry_count + 1) else: print ' failed, giving up' if not pull_status: return False return True
def __process_dockerfile(self, dockerfile): """ :param dockerfile_full_path: :type dockerfile_full_path: str :return: self :rtype: self """ if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): self.line('<info>-> Processing: </info>%s' % dockerfile['image']['fullname']) docker_image = dockerfile['image']['name'] parent_image_name = DockerfileUtility.image_basename( dockerfile['image']['from']) parent_image_tag = DockerfileUtility.extract_image_name_tag( dockerfile['image']['from']) if not parent_image_name in self.containers: self.containers[parent_image_name] = 'scratch' self.__append_tag(parent_image_name, 'latest') self.containers[docker_image] = parent_image_name self.__append_tag(docker_image, parent_image_tag) return self
def task_dependency_puller(docker_client, dockerfileList, configuration, task): """ Pulls dependency images before building """ def pull_image(image): print ' -> Pull base image %s ' % image if configuration.get('dryRun'): return True pull_image_name = DockerfileUtility.image_basename(image) pull_image_tag = DockerfileUtility.extract_image_name_tag(image) pull_status = False for retry_count in range(0, configuration.get('retry')): pull_status = docker_client.pull_image( name=pull_image_name, tag=pull_image_tag, ) if pull_status: break elif retry_count < (configuration.get('retry') - 1): print ' failed, retrying... (try %s)' % (retry_count + 1) else: print ' failed, giving up' if not pull_status: return False return True image_list = [] for dockerfile in dockerfileList: # Pull base image (FROM: xxx) first if DockerfileUtility.check_if_base_image_needs_pull( dockerfile['image']['from'], configuration): image_list.append(dockerfile['image']['from']) # Pull straged images (multi-stage dockerfiles) for multiStageImage in dockerfile['image']['multiStageImages']: if DockerfileUtility.check_if_base_image_needs_pull( multiStageImage, configuration): image_list.append(multiStageImage) # filter only unique image names image_list = set(image_list) # pull images for image in set(image_list): if not pull_image(image): return False return True
def task_run(docker_client, dockerfile, configuration, task): """ Build one Dockerfile """ pull_parent_image = DockerfileUtility.check_if_base_image_needs_pull(dockerfile, configuration) if configuration.get('dryRun'): print ' from: %s (pull: %s)' % (dockerfile['image']['from'], ('yes' if pull_parent_image else 'no')) print ' path: %s' % dockerfile['path'] print ' dep: %s' % (DockerBuildTaskLoader.human_task_name_list(task.task_dep) if task.task_dep else 'none') return True # Pull base image (FROM: xxx) first if pull_parent_image: print ' -> Pull base image %s ' % dockerfile['image']['from'] pull_image_name = DockerfileUtility.image_basename(dockerfile['image']['from']) pull_image_tag = DockerfileUtility.extract_image_name_tag(dockerfile['image']['from']) pull_status = False for retry_count in range(0, configuration.get('retry')): pull_status = docker_client.pull_image( name=pull_image_name, tag=pull_image_tag, ) if pull_status: break elif retry_count < (configuration.get('retry') - 1): print ' failed, retrying... (try %s)' % (retry_count+1) else: print ' failed, giving up' if not pull_status: return False ## Build image print ' -> Building image %s ' % dockerfile['image']['fullname'] build_status = False for retry_count in range(0, configuration.get('retry')): build_status = docker_client.build_dockerfile( path=dockerfile['path'], name=dockerfile['image']['fullname'], nocache=configuration.get('dockerBuild.noCache'), ) if build_status: break elif retry_count < (configuration.get('retry')-1): print ' failed, retrying... (try %s)' % (retry_count+1) else: print ' failed, giving up' return build_status
def task_dependency_puller(docker_client, dockerfileList, configuration, task): """ Pulls dependency images before building """ def pull_image(image): print ' -> Pull base image %s ' % image if configuration.get('dryRun'): return True pull_image_name = DockerfileUtility.image_basename(image) pull_image_tag = DockerfileUtility.extract_image_name_tag(image) pull_status = False for retry_count in range(0, configuration.get('retry')): pull_status = docker_client.pull_image( name=pull_image_name, tag=pull_image_tag, ) if pull_status: break elif retry_count < (configuration.get('retry') - 1): print ' failed, retrying... (try %s)' % (retry_count + 1) else: print ' failed, giving up' if not pull_status: return False return True image_list = [] for dockerfile in dockerfileList: # Pull base image (FROM: xxx) first if DockerfileUtility.check_if_base_image_needs_pull(dockerfile['image']['from'], configuration): image_list.append(dockerfile['image']['from']) # Pull straged images (multi-stage dockerfiles) for multiStageImage in dockerfile['image']['multiStageImages']: if DockerfileUtility.check_if_base_image_needs_pull(multiStageImage, configuration): image_list.append(multiStageImage) # filter only unique image names image_list = set(image_list) # pull images for image in set(image_list): if not pull_image(image): return False return True
def load_tasks(self, cmd, opt_values, pos_args): """ DOIT task list generator """ config = {'verbosity': self.configuration.get('verbosity')} dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( base_path=self.configuration.get('dockerPath'), path_regex=self.configuration.get('docker.pathRegex'), image_prefix=self.configuration.get('docker.imagePrefix'), whitelist=self.configuration.get('whitelist'), blacklist=self.configuration.get('blacklist'), ) dockerfile_list = self.process_dockerfile_list(dockerfile_list) #import json,sys;print json.dumps(dockerfile_list, sort_keys=True, indent = 4, separators = (',', ': '));sys.exit(0); tasklist = self.generate_task_list(dockerfile_list) if not tasklist or len(tasklist) == 0: raise Exception('No tasks found') tasklist = self.process_tasklist(tasklist) return tasklist, config
def run_task(self, configuration): template_path = configuration.get('templatePath') dockerfile_path = configuration.get('dockerPath') whitelist = self.get_whitelist() blacklist = self.get_blacklist() if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): self.line('<info>-> </info><comment>docker path</comment> : %s' % dockerfile_path) self.line('<info>-> </info><comment>template path </comment> : %s' % template_path) if whitelist: self.line('<info>-> </info><comment>whitelist </comment> :') for crit in whitelist: self.line("\t * %s" % crit) if blacklist: self.line('<info>-> </info><comment>blacklist </comment> :') for crit in blacklist: self.line("\t * %s" % crit) self.template = Environment( autoescape=False, loader=FileSystemLoader([template_path]), trim_blocks=False ) for file in DockerfileUtility.find_file_in_path(dockerfile_path=dockerfile_path, filename="Dockerfile.jinja2", whitelist=whitelist, blacklist=blacklist): self.process_dockerfile(file)
def process_dockerfile_list(self, dockerfile_list): """ Prepare dockerfile list with dependency and also add "auto latest tag" images """ image_list = [x['image']['fullname'] for x in dockerfile_list if x['image']['fullname']] autoLatestTagImageList = [] for dockerfile in dockerfile_list: # Calculate dependency dockerfile['dependency'] = False if dockerfile['image']['from'] and dockerfile['image']['from'] in image_list: dockerfile['dependency'] = dockerfile['image']['from'] # Process auto latest tag if self.configuration.get('docker.autoLatestTag') and dockerfile['image']['tag'] == self.configuration.get('docker.autoLatestTag'): imageNameLatest = DockerfileUtility.generate_image_name_with_tag_latest(dockerfile['image']['fullname']) if imageNameLatest not in image_list: autoLatestTagImage = copy.deepcopy(dockerfile) autoLatestTagImage['image']['fullname'] = imageNameLatest autoLatestTagImage['image']['tag'] = 'latest' autoLatestTagImage['dependency'] = dockerfile['image']['fullname'] autoLatestTagImageList.append(autoLatestTagImage) # Add auto latest tag images to dockerfile list dockerfile_list.extend(autoLatestTagImageList) return dockerfile_list
def run_task(self, configuration): template_path = configuration.get('templatePath') dockerfile_path = configuration.get('dockerPath') whitelist = self.get_whitelist() blacklist = self.get_blacklist() if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): self.line('<info>-> </info><comment>docker path</comment> : %s' % dockerfile_path) self.line( '<info>-> </info><comment>template path </comment> : %s' % template_path) if whitelist: self.line('<info>-> </info><comment>whitelist </comment> :') for crit in whitelist: self.line("\t * %s" % crit) if blacklist: self.line('<info>-> </info><comment>blacklist </comment> :') for crit in blacklist: self.line("\t * %s" % crit) self.template = Environment(autoescape=False, loader=FileSystemLoader([template_path]), trim_blocks=False) for file in DockerfileUtility.find_file_in_path( dockerfile_path=dockerfile_path, filename="Dockerfile.jinja2", whitelist=whitelist, blacklist=blacklist): self.process_dockerfile(file)
def run_task(self, configuration): if self.option('output'): configuration.set('imagePath', self.option('output')) if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): self.line('<info>ALL :</info> %s' % self.option('all')) self.line('<info>output :</info> %s' % configuration.get('imagePath')) self.line('<info>format :</info> %s' % self.option('format')) self.line('<info>dockerPath :</info> %s' % configuration.get('dockerPath')) self.line('<info>filename :</info> %s' % self.option('filename')) self.__load_configuration() dockerfileList = DockerfileUtility.find_dockerfiles_in_path( base_path=configuration.get('dockerPath'), path_regex=configuration.get('docker.pathRegex'), image_prefix=configuration.get('docker.imagePrefix'), ) for dockerfile in dockerfileList: self.__process_dockerfile(dockerfile) dia = self.build_graph() dia.render()
def run_task(self, configuration): dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( base_path=configuration.get('dockerPath'), path_regex=configuration.get('docker.pathRegex'), image_prefix=configuration.get('docker.imagePrefix'), whitelist=configuration.get('whitelist'), blacklist=configuration.get('blacklist'), ) image_fail_list = [] docker_command = self.argument('docker command') for dockerfile in dockerfile_list: title = dockerfile['image']['fullname'] print title print '~' * len(title) if configuration['dryRun']: print ' exec: %s' % (docker_command) else: cmd = [ 'docker', 'run', '-t', '--rm', dockerfile['image']['fullname'], 'bash', '-c', '%s' % (' '.join(docker_command)) ] status = Command.execute(cmd) if status: print colored(' -> successfull', 'green') else: print colored(' -> failed', 'red') image_fail_list.append(dockerfile['image']['fullname']) print '' print '' if len(image_fail_list) >= 1: print '' print colored(' => failed images (%s):' % (str(len(image_fail_list))), 'red') for image in image_fail_list: print ' - %s ' % image print '' return False else: return True
def run_task(self, configuration): dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( base_path=configuration.get('dockerPath'), path_regex=configuration.get('docker.pathRegex'), image_prefix=configuration.get('docker.imagePrefix'), whitelist=configuration.get('whitelist'), blacklist=configuration.get('blacklist'), ) image_fail_list = [] docker_command = self.argument('docker command') for dockerfile in dockerfile_list: title = dockerfile['image']['fullname'] print title print '~' * len(title) if configuration['dryRun']: print ' exec: %s' % (docker_command) else: cmd = [ 'docker', 'run', '-t', '--rm', dockerfile['image']['fullname'], 'bash', '-c', '%s' % (' '.join(docker_command)) ] status = Command.execute(cmd) if status: print colored(' -> successfull', 'green') else: print colored(' -> failed', 'red') image_fail_list.append(dockerfile['image']['fullname']) print '' print '' if len(image_fail_list) >= 1: print '' print colored( ' => failed images (%s):' % (str(len(image_fail_list))), 'red') for image in image_fail_list: print ' - %s ' % image print '' return False else: return True
def init_docker_image_list(self, docker_image=False): """ Init and build list of available docker images """ if docker_image: self.docker_image_list.append(docker_image) else: dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( base_path=self.configuration.get('dockerPath'), path_regex=self.configuration.get('docker.pathRegex'), image_prefix=self.configuration.get('docker.imagePrefix'), whitelist=self.configuration.get('whitelist'), blacklist=self.configuration.get('blacklist'), ) for image in dockerfile_list: self.docker_image_list.append(image['image']['fullname'])
def process_dockerfile_list(self, dockerfile_list): """ Prepare dockerfile list with dependency and also add "auto latest tag" images """ # Process auto latest tag autoLatestTagImageList = [] image_list = [x['image']['fullname'] for x in dockerfile_list if x['image']['fullname']] for dockerfile in dockerfile_list: if self.configuration.get('docker.autoLatestTag') and dockerfile['image']['tag'] == self.configuration.get('docker.autoLatestTag'): imageNameLatest = DockerfileUtility.generate_image_name_with_tag_latest(dockerfile['image']['fullname']) if imageNameLatest not in image_list: autoLatestTagImage = copy.deepcopy(dockerfile) autoLatestTagImage['image']['fullname'] = imageNameLatest autoLatestTagImage['image']['tag'] = 'latest' autoLatestTagImage['image']['duplicate'] = True if not 'dependency' in autoLatestTagImage: autoLatestTagImage['dependency'] = [] autoLatestTagImage['dependency'].append(dockerfile['image']['fullname']) autoLatestTagImageList.append(autoLatestTagImage) # Add auto latest tag images to dockerfile list dockerfile_list.extend(autoLatestTagImageList) # Calculate dependency image_list = [x['image']['fullname'] for x in dockerfile_list if x['image']['fullname']] for dockerfile in dockerfile_list: if not 'dependency' in dockerfile: dockerfile['dependency'] = [] # add image from if it is a dependency if dockerfile['image']['from'] and dockerfile['image']['from'] in image_list: dockerfile['dependency'].append(dockerfile['image']['from']) # add multi stage image if it is a dependency for multiStageImage in dockerfile['image']['multiStageImages']: if multiStageImage in image_list: dockerfile['dependency'].append(multiStageImage) # unique list unique_dep_list = [] [unique_dep_list.append(item) for item in dockerfile['dependency'] if item not in unique_dep_list] dockerfile['dependency'] = unique_dep_list return dockerfile_list
def process_dockerfile_list(self, dockerfile_list): """ Prepare dockerfile list with dependency and also add "auto latest tag" images """ # Process auto latest tag autoLatestTagImageList = [] image_list = [ x['image']['fullname'] for x in dockerfile_list if x['image']['fullname'] ] for dockerfile in dockerfile_list: if self.configuration.get('docker.autoLatestTag') and dockerfile[ 'image']['tag'] == self.configuration.get( 'docker.autoLatestTag'): imageNameLatest = DockerfileUtility.generate_image_name_with_tag_latest( dockerfile['image']['fullname']) if imageNameLatest not in image_list: autoLatestTagImage = copy.deepcopy(dockerfile) autoLatestTagImage['image']['fullname'] = imageNameLatest autoLatestTagImage['image']['tag'] = 'latest' autoLatestTagImage['dependency'] = dockerfile['image'][ 'fullname'] autoLatestTagImageList.append(autoLatestTagImage) # Add auto latest tag images to dockerfile list dockerfile_list.extend(autoLatestTagImageList) # Calculate dependency image_list = [ x['image']['fullname'] for x in dockerfile_list if x['image']['fullname'] ] for dockerfile in dockerfile_list: if not 'dependency' in dockerfile: dockerfile['dependency'] = False if dockerfile['image']['from'] and dockerfile['image'][ 'from'] in image_list: dockerfile['dependency'] = dockerfile['image']['from'] return dockerfile_list
def task_run(docker_client, dockerfile, configuration, task): """ Build one Dockerfile """ pull_parent_image = DockerfileUtility.check_if_base_image_needs_pull( dockerfile, configuration) if configuration.get('dryRun'): print ' from: %s (pull: %s)' % (dockerfile['image']['from'], ('yes' if pull_parent_image else 'no')) print ' path: %s' % dockerfile['path'] print ' dep: %s' % ( DockerBuildTaskLoader.human_task_name_list(task.task_dep) if task.task_dep else 'none') return True # Pull base image (FROM: xxx) first if pull_parent_image: print ' -> Pull base image %s ' % dockerfile['image']['from'] pull_image_name = DockerfileUtility.image_basename( dockerfile['image']['from']) pull_image_tag = DockerfileUtility.extract_image_name_tag( dockerfile['image']['from']) pull_status = False for retry_count in range(0, configuration.get('retry')): pull_status = docker_client.pull_image( name=pull_image_name, tag=pull_image_tag, ) if pull_status: break elif retry_count < (configuration.get('retry') - 1): print ' failed, retrying... (try %s)' % (retry_count + 1) else: print ' failed, giving up' if not pull_status: return False ## Build image print ' -> Building image %s ' % dockerfile['image']['fullname'] build_status = False for retry_count in range(0, configuration.get('retry')): build_status = docker_client.build_dockerfile( path=dockerfile['path'], name=dockerfile['image']['fullname'], nocache=configuration.get('dockerBuild.noCache'), ) if build_status: break elif retry_count < (configuration.get('retry') - 1): print ' failed, retrying... (try %s)' % (retry_count + 1) else: print ' failed, giving up' return build_status