Пример #1
0
 def test_storeImageUpload(self):
     image = "ubuntu-trusty"
     provider = "rax"
     bnum = self.zk.storeBuild(image, zk.ImageBuild())
     up1 = self.zk.storeImageUpload(image, bnum, provider, zk.ImageUpload())
     up2 = self.zk.storeImageUpload(image, bnum, provider, zk.ImageUpload())
     self.assertLess(int(up1), int(up2))
Пример #2
0
    def test_store_and_get_image_upload(self):
        image = "ubuntu-trusty"
        provider = "rax"
        orig_data = zk.ImageUpload()
        orig_data.external_id = "deadbeef"
        orig_data.state = zk.READY
        orig_data.format = "qcow2"

        build_number = self.zk.storeBuild(image, zk.ImageBuild())
        upload_id = self.zk.storeImageUpload(image, build_number, provider,
                                             orig_data)
        data = self.zk.getImageUpload(image, build_number, provider, upload_id)

        self.assertEqual(upload_id, data.id)
        self.assertEqual(orig_data.external_id, data.external_id)
        self.assertEqual(orig_data.state, data.state)
        self.assertEqual(orig_data.state_time, data.state_time)
        self.assertEqual(orig_data.format, data.format)
        self.assertEqual(self.zk.getBuildProviders("ubuntu-trusty",
                                                   build_number),
                         [provider])
        self.assertEqual(self.zk.getImageUploadNumbers("ubuntu-trusty",
                                                       build_number,
                                                       provider),
                         [upload_id])
Пример #3
0
    def test_ImageBuild_toDict(self):
        o = zk.ImageBuild('0001')
        o.builder = 'localhost'
        o.formats = ['qemu', 'raw']

        d = o.toDict()
        self.assertNotIn('id', d)
        self.assertEqual(','.join(o.formats), d['formats'])
        self.assertEqual(o.builder, d['builder'])
Пример #4
0
    def test_getBuilds(self):
        image = "ubuntu-trusty"
        path = self.zk._imageBuildsPath(image)
        v1 = zk.ImageBuild()
        v1.state = zk.READY
        v2 = zk.ImageBuild()
        v2.state = zk.BUILDING
        v3 = zk.ImageBuild()
        v3.state = zk.FAILED
        v4 = zk.ImageBuild()
        v4.state = zk.DELETING
        self.zk.client.create(path + "/1", value=v1.serialize(), makepath=True)
        self.zk.client.create(path + "/2", value=v2.serialize(), makepath=True)
        self.zk.client.create(path + "/3", value=v3.serialize(), makepath=True)
        self.zk.client.create(path + "/4", value=v4.serialize(), makepath=True)
        self.zk.client.create(path + "/lock", makepath=True)

        matches = self.zk.getBuilds(image, [zk.DELETING, zk.FAILED])
        self.assertEqual(2, len(matches))
Пример #5
0
    def _checkImageForScheduledImageUpdates(self, diskimage):
        '''
        Check one DIB image to see if it needs to be rebuilt.

        .. note:: It's important to lock the image build before we check
            the state time and then build to eliminate any race condition.
        '''
        # Check if diskimage builds are paused.
        if diskimage.pause:
            return

        if not diskimage.image_types:
            # We don't know what formats to build.
            return

        now = int(time.time())
        builds = self._zk.getMostRecentBuilds(1, diskimage.name, zk.READY)

        # If there is no build for this image, or it has aged out
        # or if the current build is missing an image type from
        # the config file, start a new build.
        if (not builds
            or (now - builds[0].state_time) >= diskimage.rebuild_age
            or not set(builds[0].formats).issuperset(diskimage.image_types)
            ):

            try:
                with self._zk.imageBuildLock(diskimage.name, blocking=False):
                    # To avoid locking each image repeatedly, we have an
                    # second, redundant check here to verify that a new
                    # build didn't appear between the first check and the
                    # lock acquisition. If it's not the same build as
                    # identified in the first check above, assume another
                    # BuildWorker created the build for us and continue.
                    builds2 = self._zk.getMostRecentBuilds(
                        1, diskimage.name, zk.READY)
                    if builds2 and builds[0].id != builds2[0].id:
                        return

                    self.log.info("Building image %s" % diskimage.name)

                    data = zk.ImageBuild()
                    data.state = zk.BUILDING
                    data.builder_id = self._builder_id
                    data.builder = self._hostname
                    data.formats = list(diskimage.image_types)

                    bnum = self._zk.storeBuild(diskimage.name, data)
                    data = self._buildImage(bnum, diskimage)
                    self._zk.storeBuild(diskimage.name, data, bnum)
            except exceptions.ZKLockException:
                # Lock is already held. Skip it.
                pass
Пример #6
0
    def test_deleteBuild_with_uploads(self):
        image = 'trusty'
        provider = 'rax'

        build = zk.ImageBuild()
        build.state = zk.READY
        bnum = self.zk.storeBuild(image, build)

        upload = zk.ImageUpload()
        upload.state = zk.READY
        self.zk.storeImageUpload(image, bnum, provider, upload)

        self.assertFalse(self.zk.deleteBuild(image, bnum))
Пример #7
0
    def test_ImageBuild_toDict(self):
        o = zk.ImageBuild('0001')
        o.state = zk.BUILDING
        o.builder = 'localhost'
        o.builder_id = 'ABC-123'
        o.formats = ['qemu', 'raw']

        d = o.toDict()
        self.assertNotIn('id', d)
        self.assertEqual(o.state, d['state'])
        self.assertIsNotNone(d['state_time'])
        self.assertEqual(','.join(o.formats), d['formats'])
        self.assertEqual(o.builder, d['builder'])
        self.assertEqual(o.builder_id, d['builder_id'])
Пример #8
0
    def test_getMostRecentImageUpload(self):
        image = "ubuntu-trusty"
        provider = "rax"

        build1 = zk.ImageBuild()
        build1.state = zk.READY
        build2 = zk.ImageBuild()
        build2.state = zk.READY
        build2.state_time = build1.state_time + 10

        bnum1 = self.zk.storeBuild(image, build1)
        bnum2 = self.zk.storeBuild(image, build2)

        upload1 = zk.ImageUpload()
        upload1.state = zk.READY
        upload2 = zk.ImageUpload()
        upload2.state = zk.READY
        upload2.state_time = upload1.state_time + 10

        self.zk.storeImageUpload(image, bnum1, provider, upload1)
        self.zk.storeImageUpload(image, bnum2, provider, upload2)

        d = self.zk.getMostRecentImageUpload(image, provider, zk.READY)
        self.assertEqual(upload2.state_time, d.state_time)
Пример #9
0
    def test_store_and_get_build(self):
        image = "ubuntu-trusty"
        orig_data = zk.ImageBuild()
        orig_data.builder = 'host'
        orig_data.state = zk.READY
        with self.zk.imageBuildLock(image, blocking=True, timeout=1):
            build_num = self.zk.storeBuild(image, orig_data)

        data = self.zk.getBuild(image, build_num)
        self.assertEqual(orig_data.builder, data.builder)
        self.assertEqual(orig_data.state, data.state)
        self.assertEqual(orig_data.state_time, data.state_time)
        self.assertEqual(build_num, data.id)
        self.assertEqual(self.zk.getImageNames(), ["ubuntu-trusty"])
        self.assertEqual(self.zk.getBuildNumbers("ubuntu-trusty"), [build_num])
Пример #10
0
    def _checkImageForManualBuildRequest(self, diskimage):
        '''
        Query ZooKeeper for a manual image build request for one image.
        '''
        # Check if diskimage builds are paused.
        if diskimage.pause:
            return

        # Reduce use of locks by adding an initial check here and
        # a redundant check after lock acquisition.
        if not self._zk.hasBuildRequest(diskimage.name):
            return

        try:
            with self._zk.imageBuildLock(diskimage.name, blocking=False):
                # Redundant check
                if not self._zk.hasBuildRequest(diskimage.name):
                    return

                self.log.info(
                    "Manual build request for image %s" % diskimage.name)

                data = zk.ImageBuild()
                data.state = zk.BUILDING
                data.builder_id = self._builder_id
                data.builder = self._hostname
                data.formats = list(diskimage.image_types)

                bnum = self._zk.storeBuild(diskimage.name, data)
                data = self._buildImage(bnum, diskimage)
                self._zk.storeBuild(diskimage.name, data, bnum)

                # Remove request on a successful build
                if data.state == zk.READY:
                    self._zk.removeBuildRequest(diskimage.name)

        except exceptions.ZKLockException:
            # Lock is already held. Skip it.
            pass
Пример #11
0
 def test_deleteBuild(self):
     image = 'trusty'
     build = zk.ImageBuild()
     build.state = zk.READY
     bnum = self.zk.storeBuild(image, build)
     self.assertTrue(self.zk.deleteBuild(image, bnum))
Пример #12
0
 def test_storeBuild(self):
     image = "ubuntu-trusty"
     b1 = self.zk.storeBuild(image, zk.ImageBuild())
     b2 = self.zk.storeBuild(image, zk.ImageBuild())
     self.assertLess(int(b1), int(b2))
Пример #13
0
    def _buildImage(self, build_id, diskimage):
        '''
        Run the external command to build the diskimage.

        :param str build_id: The ID for the build (used in image filename).
        :param diskimage: The diskimage as retrieved from our config file.

        :returns: An ImageBuild object of build-related data.

        :raises: BuilderError if we failed to execute the build command.
        '''
        base = "-".join([diskimage.name, build_id])
        image_file = DibImageFile(base)
        filename = image_file.to_path(self._config.imagesdir, False)

        env = os.environ.copy()
        env['DIB_RELEASE'] = diskimage.release
        env['DIB_IMAGE_NAME'] = diskimage.name
        env['DIB_IMAGE_FILENAME'] = filename

        # Note we use a reference to the nodepool config here so
        # that whenever the config is updated we get up to date
        # values in this thread.
        if self._config.elementsdir:
            env['ELEMENTS_PATH'] = self._config.elementsdir

        # send additional env vars if needed
        for k, v in diskimage.env_vars.items():
            env[k] = v

        img_elements = diskimage.elements
        img_types = ",".join(diskimage.image_types)

        qemu_img_options = ''
        if 'qcow2' in img_types:
            qemu_img_options = DEFAULT_QEMU_IMAGE_COMPAT_OPTIONS

        cmd = ('%s -x -t %s --checksum --no-tmpfs %s -o %s %s' %
               (self.dib_cmd, img_types, qemu_img_options, filename,
                img_elements))

        self._pruneBuildLogs(diskimage.name)
        log_fn = self._getBuildLog(diskimage.name, build_id)

        self.log.info('Running %s' % (cmd,))
        self.log.info('Logging to %s' % (log_fn,))

        start_time = time.monotonic()
        try:
            p = subprocess.Popen(
                shlex.split(cmd),
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                env=env)
        except OSError as e:
            raise exceptions.BuilderError(
                "Failed to exec '%s'. Error: '%s'" % (cmd, e.strerror)
            )

        with open(log_fn, 'wb') as log:
            while True:
                ln = p.stdout.readline()
                log.write(ln)
                log.flush()
                if not ln:
                    break

            rc = p.wait()
            m = "Exit code: %s\n" % rc
            log.write(m.encode('utf8'))

        # It's possible the connection to the ZK cluster could have been
        # interrupted during the build. If so, wait for it to return.
        # It could transition directly from SUSPENDED to CONNECTED, or go
        # through the LOST state before CONNECTED.
        did_suspend = False
        while self._zk.suspended or self._zk.lost:
            did_suspend = True
            self.log.info("ZooKeeper suspended during build. Waiting")
            time.sleep(SUSPEND_WAIT_TIME)
        if did_suspend:
            self.log.info("ZooKeeper available. Resuming")

        build_time = time.monotonic() - start_time

        build_data = zk.ImageBuild()
        build_data.builder_id = self._builder_id
        build_data.builder = self._hostname
        build_data.username = diskimage.username

        if self._zk.didLoseConnection:
            self.log.info("ZooKeeper lost while building %s" % diskimage.name)
            self._zk.resetLostFlag()
            build_data.state = zk.FAILED
        elif p.returncode:
            self.log.info(
                "DIB failed creating %s (%s)" % (diskimage.name, p.returncode))
            build_data.state = zk.FAILED
        else:
            self.log.info("DIB image %s is built" % diskimage.name)
            build_data.state = zk.READY
            build_data.formats = list(diskimage.image_types)

            if self._statsd:
                # record stats on the size of each image we create
                for ext in img_types.split(','):
                    key = 'nodepool.dib_image_build.%s.%s.size' % (
                        diskimage.name, ext)
                    # A bit tricky because these image files may be sparse
                    # files; we only want the true size of the file for
                    # purposes of watching if we've added too much stuff
                    # into the image.  Note that st_blocks is defined as
                    # 512-byte blocks by stat(2)
                    size = os.stat("%s.%s" % (filename, ext)).st_blocks * 512
                    self.log.debug("%s created image %s.%s (size: %d) " %
                                   (diskimage.name, filename, ext, size))
                    self._statsd.gauge(key, size)

        if self._statsd:
            # report result to statsd
            for ext in img_types.split(','):
                key_base = 'nodepool.dib_image_build.%s.%s' % (
                    diskimage.name, ext)
                self._statsd.gauge(key_base + '.rc', rc)
                self._statsd.timing(key_base + '.duration',
                                    int(build_time * 1000))

        return build_data