def load_images(data_center, prefix='../data/images'): data_dir = '%s/%s' % (prefix, data_center) with open('%s/layers.json' % data_dir) as f: layers = {l['digest']: l for l in json.load(f)} with open('%s/images.json' % data_dir) as f: images = {} for i in json.load(f): img = pb2.Image(repo=i['repo'], tag=i['tag'], parent=i['parent'], aliases=i['aliases'], layers=[ pb2.Layer(digest=dgst, size=layers[dgst]['size']) for dgst in i['layers'] ]) images[Image.ID(img.repo, img.tag)] = img return images
def _build_images( self, images: Iterable[pb2.Image], existed: Iterable[pb2.Image], layers: Dict[str, Layer], ) -> List[Image]: to_build, refs, exist = {}, {}, set() for i in existed: img = Image(i.repo, i.tag) aliases = [str(img)] + list(i.aliases) for a in aliases: refs[a] = img exist.add(a) for i in images: ls = [layers[l.digest] for l in i.layers if l.digest in layers] img = Image(i.repo, i.tag, aliases=i.aliases, layers=ls) img_id = str(img) to_build[img_id] = img aliases = [img_id] + list(i.aliases) for a in aliases: refs[a] = img for i in images: if i.parent: to_build[Image.ID(i.repo, i.tag)].parent = refs.get( i.parent, Image(*i.parent.split(':'))) # elect a major image to build and tag other aliases elected = set() for i in dict(to_build).values(): elected.add(str(i)) for a in i.aliases: if a not in elected: to_build.pop(a, None) def _build(idx, total, image): with sema: logging.info('[%d/%d] Building image %s ...' % (idx, total, image)) image.build(self.__build_dir, self.registry) logging.info('[%d/%d] Pushing image %s to %s ...' % (idx, total, image, self.registry)) image.push(self.registry) logging.info('[%d/%d] Deleting image %s ...' % (idx, total, image)) image.prune(self.registry) if len(image.aliases) > 0: logging.info( '[%d/%d] Pushing %d aliases of %s to %s ...' % (idx, total, len(image.aliases), image, self.registry)) image.push_aliases(self.registry) logging.info('Building a total of %d images' % len(to_build)) built = set() ready = [ i for i in to_build.values() if not i.parent or str(i.parent) in built or str(i.parent) in exist ] while ready: procs = [ mp.Process(target=_build, args=( len(built) + idx + 1, len(to_build), i, )) for idx, i in enumerate(ready) ] sema = mp.Semaphore( min(int(mp.cpu_count() * self.__scaling_factor), len(ready))) logging.info('Building %d images' % len(procs)) try: for p in procs: p.start() for p in procs: p.join() finally: for p in procs: if p.is_alive(): p.terminate() for i in ready: built.add(str(i)) del to_build[str(i)] ready = [ i for i in to_build.values() if not i.parent or str(i.parent) in built or str(i.parent) in exist ] logging.info('Built %d images' % len(built)) return list(to_build.values())