def uploadImage(self, image_id, provider_name, image_name): start_time = time.time() timestamp = int(start_time) provider = self._config.providers[provider_name] image_type = provider.image_type image_files = DibImageFile.from_image_id(self._config.imagesdir, image_id) for f in image_files: self.log.debug("Found image file of type %s for image id: %s" % (f.extension, f.image_id)) image_files = filter(lambda x: x.extension == image_type, image_files) if len(image_files) == 0: raise exceptions.BuilderInvalidCommandError( "Unable to find image file of type %s for id %s to upload" % (image_type, image_id) ) if len(image_files) > 1: raise exceptions.BuilderError( "Found more than one image for id %s. This should never " "happen.", image_id ) image_file = image_files[0] filename = image_file.to_path(self._config.imagesdir, with_extension=True) dummy_image = type('obj', (object,), {'name': image_name}) ext_image_name = provider.template_hostname.format( provider=provider, image=dummy_image, timestamp=str(timestamp)) self.log.info("Uploading dib image id: %s from %s in %s" % (image_id, filename, provider.name)) manager = self._config.provider_managers[provider.name] try: provider_image = provider.images[image_name] except KeyError: raise exceptions.BuilderInvalidCommandError( "Could not find matching provider image for %s", image_name ) image_meta = provider_image.meta # uploadImage is synchronous external_id = manager.uploadImage( ext_image_name, filename, image_type=image_file.extension, meta=image_meta) if self.statsd: dt = int((time.time() - start_time) * 1000) key = 'nodepool.image_update.%s.%s' % (image_name, provider.name) self.statsd.timing(key, dt) self.statsd.incr(key) self.log.info("Image id: %s in %s is ready" % (image_id, provider.name)) return external_id
def _handleJob(self, job): try: self.nplog.debug('Got job %s with data %s', job.name, job.arguments) if job.name.startswith('image-build:'): args = json.loads(job.arguments) image_id = args['image-id'] if '/' in image_id: raise exceptions.BuilderInvalidCommandError( 'Invalid image-id specified' ) image_name = job.name.split(':', 1)[1] try: self.builder.buildImage(image_name, image_id) except exceptions.BuilderError: self.nplog.exception('Exception while building image') job.sendWorkFail() else: # We can now upload this image self.builder.registerImageId(image_id) job.sendWorkComplete(json.dumps({'image-id': image_id})) else: self.nplog.error('Unable to handle job %s', job.name) job.sendWorkFail() except Exception: self.nplog.exception('Exception while running job') job.sendWorkException(traceback.format_exc())
def buildImage(self, image_name, image_id): diskimage = self._getDiskimageByName(image_name) if diskimage is None: raise exceptions.BuilderInvalidCommandError( 'Could not find matching image in config for %s', image_name) start_time = time.time() image_file = DibImageFile(image_id) filename = image_file.to_path(self._config.imagesdir, False) self.log.info("Creating image %s with filename %s" % (diskimage.name, filename)) self._runDibForImage(diskimage, filename) self.log.info("DIB image %s with filename %s is built" % (image_name, filename)) if self.statsd: dt = int((time.time() - start_time) * 1000) key = 'nodepool.dib_image_build.%s' % diskimage.name self.statsd.timing(key, dt) self.statsd.incr(key)
def _uploadImage(self, build_id, upload_id, image_name, images, provider): ''' Upload a local DIB image build to a provider. :param str build_id: Unique ID of the image build to upload. :param str upload_id: Unique ID of the upload. :param str image_name: Name of the diskimage. :param list images: A list of DibImageFile objects from this build that available for uploading. :param provider: The provider from the parsed config file. ''' start_time = time.time() timestamp = int(start_time) image = None for i in images: if provider.image_type == i.extension: image = i break if not image: raise exceptions.BuilderInvalidCommandError( "Unable to find image file of type %s for id %s to upload" % (provider.image_type, build_id)) self.log.debug("Found image file of type %s for image id: %s" % (image.extension, image.image_id)) filename = image.to_path(self._config.imagesdir, with_extension=True) dummy_image = type('obj', (object, ), { 'name': image_name, 'id': image.image_id }) ext_image_name = provider.template_hostname.format( provider=provider, image=dummy_image, timestamp=str(timestamp)) self.log.info("Uploading DIB image build %s from %s to %s" % (build_id, filename, provider.name)) manager = self._config.provider_managers[provider.name] provider_image = provider.images.get(image_name) if provider_image is None: raise exceptions.BuilderInvalidCommandError( "Could not find matching provider image for %s" % image_name) meta = provider_image.meta.copy() meta['nodepool_build_id'] = build_id meta['nodepool_upload_id'] = upload_id try: external_id = manager.uploadImage( ext_image_name, filename, image_type=image.extension, meta=meta, md5=image.md5, sha256=image.sha256, ) except Exception: self.log.exception("Failed to upload image %s to provider %s" % (image_name, provider.name)) data = zk.ImageUpload() data.state = zk.FAILED return data if self._statsd: dt = int((time.time() - start_time) * 1000) key = 'nodepool.image_update.%s.%s' % (image_name, provider.name) self._statsd.timing(key, dt) self._statsd.incr(key) base = "-".join([image_name, build_id]) self.log.info("Image build %s in %s is ready" % (base, provider.name)) data = zk.ImageUpload() data.state = zk.READY data.external_id = external_id data.external_name = ext_image_name return data