def load_images(data_center, prefix='../data/images'):
data_dir = '%s/%s' % (prefix, data_center)
with open('%s/layers.json' % data_dir) as f:
layers = {l['digest']: l for l in json.load(f)}
with open('%s/images.json' % data_dir) as f:
images = {}
for i in json.load(f):
img = pb2.Image(repo=i['repo'],
tag=i['tag'],
parent=i['parent'],
aliases=i['aliases'],
layers=[
pb2.Layer(digest=dgst,
size=layers[dgst]['size'])
for dgst in i['layers']
])
images[Image.ID(img.repo, img.tag)] = img
return images
def _build_images(
self,
images: Iterable[pb2.Image],
existed: Iterable[pb2.Image],
layers: Dict[str, Layer],
) -> List[Image]:
to_build, refs, exist = {}, {}, set()
for i in existed:
img = Image(i.repo, i.tag)
aliases = [str(img)] + list(i.aliases)
for a in aliases:
refs[a] = img
exist.add(a)
for i in images:
ls = [layers[l.digest] for l in i.layers if l.digest in layers]
img = Image(i.repo, i.tag, aliases=i.aliases, layers=ls)
img_id = str(img)
to_build[img_id] = img
aliases = [img_id] + list(i.aliases)
for a in aliases:
refs[a] = img
for i in images:
if i.parent:
to_build[Image.ID(i.repo, i.tag)].parent = refs.get(
i.parent, Image(*i.parent.split(':')))
# elect a major image to build and tag other aliases
elected = set()
for i in dict(to_build).values():
elected.add(str(i))
for a in i.aliases:
if a not in elected:
to_build.pop(a, None)
def _build(idx, total, image):
with sema:
logging.info('[%d/%d] Building image %s ...' %
(idx, total, image))
image.build(self.__build_dir, self.registry)
logging.info('[%d/%d] Pushing image %s to %s ...' %
(idx, total, image, self.registry))
image.push(self.registry)
logging.info('[%d/%d] Deleting image %s ...' %
(idx, total, image))
image.prune(self.registry)
if len(image.aliases) > 0:
logging.info(
'[%d/%d] Pushing %d aliases of %s to %s ...' %
(idx, total, len(image.aliases), image, self.registry))
image.push_aliases(self.registry)
logging.info('Building a total of %d images' % len(to_build))
built = set()
ready = [
i for i in to_build.values()
if not i.parent or str(i.parent) in built or str(i.parent) in exist
]
while ready:
procs = [
mp.Process(target=_build,
args=(
len(built) + idx + 1,
len(to_build),
i,
)) for idx, i in enumerate(ready)
]
sema = mp.Semaphore(
min(int(mp.cpu_count() * self.__scaling_factor), len(ready)))
logging.info('Building %d images' % len(procs))
try:
for p in procs:
p.start()
for p in procs:
p.join()
finally:
for p in procs:
if p.is_alive():
p.terminate()
for i in ready:
built.add(str(i))
del to_build[str(i)]
ready = [
i for i in to_build.values() if not i.parent
or str(i.parent) in built or str(i.parent) in exist
]
logging.info('Built %d images' % len(built))
return list(to_build.values())