def test_storeImageUpload(self):
image = "ubuntu-trusty"
provider = "rax"
bnum = self.zk.storeBuild(image, zk.ImageBuild())
up1 = self.zk.storeImageUpload(image, bnum, provider, zk.ImageUpload())
up2 = self.zk.storeImageUpload(image, bnum, provider, zk.ImageUpload())
self.assertLess(int(up1), int(up2))
def test_store_and_get_image_upload(self):
image = "ubuntu-trusty"
provider = "rax"
orig_data = zk.ImageUpload()
orig_data.external_id = "deadbeef"
orig_data.state = zk.READY
orig_data.format = "qcow2"
build_number = self.zk.storeBuild(image, zk.ImageBuild())
upload_id = self.zk.storeImageUpload(image, build_number, provider,
orig_data)
data = self.zk.getImageUpload(image, build_number, provider, upload_id)
self.assertEqual(upload_id, data.id)
self.assertEqual(orig_data.external_id, data.external_id)
self.assertEqual(orig_data.state, data.state)
self.assertEqual(orig_data.state_time, data.state_time)
self.assertEqual(orig_data.format, data.format)
self.assertEqual(self.zk.getBuildProviders("ubuntu-trusty",
build_number),
[provider])
self.assertEqual(self.zk.getImageUploadNumbers("ubuntu-trusty",
build_number,
provider),
[upload_id])
def test_ImageBuild_toDict(self):
o = zk.ImageBuild('0001')
o.builder = 'localhost'
o.formats = ['qemu', 'raw']
d = o.toDict()
self.assertNotIn('id', d)
self.assertEqual(','.join(o.formats), d['formats'])
self.assertEqual(o.builder, d['builder'])
def test_getBuilds(self):
image = "ubuntu-trusty"
path = self.zk._imageBuildsPath(image)
v1 = zk.ImageBuild()
v1.state = zk.READY
v2 = zk.ImageBuild()
v2.state = zk.BUILDING
v3 = zk.ImageBuild()
v3.state = zk.FAILED
v4 = zk.ImageBuild()
v4.state = zk.DELETING
self.zk.client.create(path + "/1", value=v1.serialize(), makepath=True)
self.zk.client.create(path + "/2", value=v2.serialize(), makepath=True)
self.zk.client.create(path + "/3", value=v3.serialize(), makepath=True)
self.zk.client.create(path + "/4", value=v4.serialize(), makepath=True)
self.zk.client.create(path + "/lock", makepath=True)
matches = self.zk.getBuilds(image, [zk.DELETING, zk.FAILED])
self.assertEqual(2, len(matches))
def _checkImageForScheduledImageUpdates(self, diskimage):
'''
Check one DIB image to see if it needs to be rebuilt.
.. note:: It's important to lock the image build before we check
the state time and then build to eliminate any race condition.
'''
# Check if diskimage builds are paused.
if diskimage.pause:
return
if not diskimage.image_types:
# We don't know what formats to build.
return
now = int(time.time())
builds = self._zk.getMostRecentBuilds(1, diskimage.name, zk.READY)
# If there is no build for this image, or it has aged out
# or if the current build is missing an image type from
# the config file, start a new build.
if (not builds
or (now - builds[0].state_time) >= diskimage.rebuild_age
or not set(builds[0].formats).issuperset(diskimage.image_types)
):
try:
with self._zk.imageBuildLock(diskimage.name, blocking=False):
# To avoid locking each image repeatedly, we have an
# second, redundant check here to verify that a new
# build didn't appear between the first check and the
# lock acquisition. If it's not the same build as
# identified in the first check above, assume another
# BuildWorker created the build for us and continue.
builds2 = self._zk.getMostRecentBuilds(
1, diskimage.name, zk.READY)
if builds2 and builds[0].id != builds2[0].id:
return
self.log.info("Building image %s" % diskimage.name)
data = zk.ImageBuild()
data.state = zk.BUILDING
data.builder_id = self._builder_id
data.builder = self._hostname
data.formats = list(diskimage.image_types)
bnum = self._zk.storeBuild(diskimage.name, data)
data = self._buildImage(bnum, diskimage)
self._zk.storeBuild(diskimage.name, data, bnum)
except exceptions.ZKLockException:
# Lock is already held. Skip it.
pass
def test_deleteBuild_with_uploads(self):
image = 'trusty'
provider = 'rax'
build = zk.ImageBuild()
build.state = zk.READY
bnum = self.zk.storeBuild(image, build)
upload = zk.ImageUpload()
upload.state = zk.READY
self.zk.storeImageUpload(image, bnum, provider, upload)
self.assertFalse(self.zk.deleteBuild(image, bnum))
def test_ImageBuild_toDict(self):
o = zk.ImageBuild('0001')
o.state = zk.BUILDING
o.builder = 'localhost'
o.builder_id = 'ABC-123'
o.formats = ['qemu', 'raw']
d = o.toDict()
self.assertNotIn('id', d)
self.assertEqual(o.state, d['state'])
self.assertIsNotNone(d['state_time'])
self.assertEqual(','.join(o.formats), d['formats'])
self.assertEqual(o.builder, d['builder'])
self.assertEqual(o.builder_id, d['builder_id'])
def test_getMostRecentImageUpload(self):
image = "ubuntu-trusty"
provider = "rax"
build1 = zk.ImageBuild()
build1.state = zk.READY
build2 = zk.ImageBuild()
build2.state = zk.READY
build2.state_time = build1.state_time + 10
bnum1 = self.zk.storeBuild(image, build1)
bnum2 = self.zk.storeBuild(image, build2)
upload1 = zk.ImageUpload()
upload1.state = zk.READY
upload2 = zk.ImageUpload()
upload2.state = zk.READY
upload2.state_time = upload1.state_time + 10
self.zk.storeImageUpload(image, bnum1, provider, upload1)
self.zk.storeImageUpload(image, bnum2, provider, upload2)
d = self.zk.getMostRecentImageUpload(image, provider, zk.READY)
self.assertEqual(upload2.state_time, d.state_time)
def test_store_and_get_build(self):
image = "ubuntu-trusty"
orig_data = zk.ImageBuild()
orig_data.builder = 'host'
orig_data.state = zk.READY
with self.zk.imageBuildLock(image, blocking=True, timeout=1):
build_num = self.zk.storeBuild(image, orig_data)
data = self.zk.getBuild(image, build_num)
self.assertEqual(orig_data.builder, data.builder)
self.assertEqual(orig_data.state, data.state)
self.assertEqual(orig_data.state_time, data.state_time)
self.assertEqual(build_num, data.id)
self.assertEqual(self.zk.getImageNames(), ["ubuntu-trusty"])
self.assertEqual(self.zk.getBuildNumbers("ubuntu-trusty"), [build_num])
def _checkImageForManualBuildRequest(self, diskimage):
'''
Query ZooKeeper for a manual image build request for one image.
'''
# Check if diskimage builds are paused.
if diskimage.pause:
return
# Reduce use of locks by adding an initial check here and
# a redundant check after lock acquisition.
if not self._zk.hasBuildRequest(diskimage.name):
return
try:
with self._zk.imageBuildLock(diskimage.name, blocking=False):
# Redundant check
if not self._zk.hasBuildRequest(diskimage.name):
return
self.log.info(
"Manual build request for image %s" % diskimage.name)
data = zk.ImageBuild()
data.state = zk.BUILDING
data.builder_id = self._builder_id
data.builder = self._hostname
data.formats = list(diskimage.image_types)
bnum = self._zk.storeBuild(diskimage.name, data)
data = self._buildImage(bnum, diskimage)
self._zk.storeBuild(diskimage.name, data, bnum)
# Remove request on a successful build
if data.state == zk.READY:
self._zk.removeBuildRequest(diskimage.name)
except exceptions.ZKLockException:
# Lock is already held. Skip it.
pass
def test_deleteBuild(self):
image = 'trusty'
build = zk.ImageBuild()
build.state = zk.READY
bnum = self.zk.storeBuild(image, build)
self.assertTrue(self.zk.deleteBuild(image, bnum))
def test_storeBuild(self):
image = "ubuntu-trusty"
b1 = self.zk.storeBuild(image, zk.ImageBuild())
b2 = self.zk.storeBuild(image, zk.ImageBuild())
self.assertLess(int(b1), int(b2))
def _buildImage(self, build_id, diskimage):
'''
Run the external command to build the diskimage.
:param str build_id: The ID for the build (used in image filename).
:param diskimage: The diskimage as retrieved from our config file.
:returns: An ImageBuild object of build-related data.
:raises: BuilderError if we failed to execute the build command.
'''
base = "-".join([diskimage.name, build_id])
image_file = DibImageFile(base)
filename = image_file.to_path(self._config.imagesdir, False)
env = os.environ.copy()
env['DIB_RELEASE'] = diskimage.release
env['DIB_IMAGE_NAME'] = diskimage.name
env['DIB_IMAGE_FILENAME'] = filename
# Note we use a reference to the nodepool config here so
# that whenever the config is updated we get up to date
# values in this thread.
if self._config.elementsdir:
env['ELEMENTS_PATH'] = self._config.elementsdir
# send additional env vars if needed
for k, v in diskimage.env_vars.items():
env[k] = v
img_elements = diskimage.elements
img_types = ",".join(diskimage.image_types)
qemu_img_options = ''
if 'qcow2' in img_types:
qemu_img_options = DEFAULT_QEMU_IMAGE_COMPAT_OPTIONS
cmd = ('%s -x -t %s --checksum --no-tmpfs %s -o %s %s' %
(self.dib_cmd, img_types, qemu_img_options, filename,
img_elements))
self._pruneBuildLogs(diskimage.name)
log_fn = self._getBuildLog(diskimage.name, build_id)
self.log.info('Running %s' % (cmd,))
self.log.info('Logging to %s' % (log_fn,))
start_time = time.monotonic()
try:
p = subprocess.Popen(
shlex.split(cmd),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
except OSError as e:
raise exceptions.BuilderError(
"Failed to exec '%s'. Error: '%s'" % (cmd, e.strerror)
)
with open(log_fn, 'wb') as log:
while True:
ln = p.stdout.readline()
log.write(ln)
log.flush()
if not ln:
break
rc = p.wait()
m = "Exit code: %s\n" % rc
log.write(m.encode('utf8'))
# It's possible the connection to the ZK cluster could have been
# interrupted during the build. If so, wait for it to return.
# It could transition directly from SUSPENDED to CONNECTED, or go
# through the LOST state before CONNECTED.
did_suspend = False
while self._zk.suspended or self._zk.lost:
did_suspend = True
self.log.info("ZooKeeper suspended during build. Waiting")
time.sleep(SUSPEND_WAIT_TIME)
if did_suspend:
self.log.info("ZooKeeper available. Resuming")
build_time = time.monotonic() - start_time
build_data = zk.ImageBuild()
build_data.builder_id = self._builder_id
build_data.builder = self._hostname
build_data.username = diskimage.username
if self._zk.didLoseConnection:
self.log.info("ZooKeeper lost while building %s" % diskimage.name)
self._zk.resetLostFlag()
build_data.state = zk.FAILED
elif p.returncode:
self.log.info(
"DIB failed creating %s (%s)" % (diskimage.name, p.returncode))
build_data.state = zk.FAILED
else:
self.log.info("DIB image %s is built" % diskimage.name)
build_data.state = zk.READY
build_data.formats = list(diskimage.image_types)
if self._statsd:
# record stats on the size of each image we create
for ext in img_types.split(','):
key = 'nodepool.dib_image_build.%s.%s.size' % (
diskimage.name, ext)
# A bit tricky because these image files may be sparse
# files; we only want the true size of the file for
# purposes of watching if we've added too much stuff
# into the image. Note that st_blocks is defined as
# 512-byte blocks by stat(2)
size = os.stat("%s.%s" % (filename, ext)).st_blocks * 512
self.log.debug("%s created image %s.%s (size: %d) " %
(diskimage.name, filename, ext, size))
self._statsd.gauge(key, size)
if self._statsd:
# report result to statsd
for ext in img_types.split(','):
key_base = 'nodepool.dib_image_build.%s.%s' % (
diskimage.name, ext)
self._statsd.gauge(key_base + '.rc', rc)
self._statsd.timing(key_base + '.duration',
int(build_time * 1000))
return build_data