def create_image(self, **kwargs):
if self.times_to_fail is None:
raise exceptions.BuilderError("Test fail image upload.")
self.times_failed += 1
if self.times_failed <= self.times_to_fail:
raise exceptions.BuilderError("Test fail image upload.")
else:
return super(FakeUploadFailCloud, self).create_image(**kwargs)
def uploadImage(self, image_id, provider_name, image_name):
start_time = time.time()
timestamp = int(start_time)
provider = self._config.providers[provider_name]
image_type = provider.image_type
image_files = DibImageFile.from_image_id(self._config.imagesdir,
image_id)
for f in image_files:
self.log.debug("Found image file of type %s for image id: %s" %
(f.extension, f.image_id))
image_files = filter(lambda x: x.extension == image_type, image_files)
if len(image_files) == 0:
raise exceptions.BuilderInvalidCommandError(
"Unable to find image file of type %s for id %s to upload" %
(image_type, image_id)
)
if len(image_files) > 1:
raise exceptions.BuilderError(
"Found more than one image for id %s. This should never "
"happen.", image_id
)
image_file = image_files[0]
filename = image_file.to_path(self._config.imagesdir,
with_extension=True)
dummy_image = type('obj', (object,),
{'name': image_name})
ext_image_name = provider.template_hostname.format(
provider=provider, image=dummy_image, timestamp=str(timestamp))
self.log.info("Uploading dib image id: %s from %s in %s" %
(image_id, filename, provider.name))
manager = self._config.provider_managers[provider.name]
try:
provider_image = provider.images[image_name]
except KeyError:
raise exceptions.BuilderInvalidCommandError(
"Could not find matching provider image for %s", image_name
)
image_meta = provider_image.meta
# uploadImage is synchronous
external_id = manager.uploadImage(
ext_image_name, filename,
image_type=image_file.extension,
meta=image_meta)
if self.statsd:
dt = int((time.time() - start_time) * 1000)
key = 'nodepool.image_update.%s.%s' % (image_name,
provider.name)
self.statsd.timing(key, dt)
self.statsd.incr(key)
self.log.info("Image id: %s in %s is ready" % (image_id,
provider.name))
return external_id
def to_path(self, images_dir, with_extension=True):
my_path = os.path.join(images_dir, self.image_id)
if with_extension:
if self.extension is None:
raise exceptions.BuilderError(
'Cannot specify image extension of None')
my_path += '.' + self.extension
return my_path
def start(self):
'''
Start the builder.
The builder functionality is encapsulated within threads run
by the NodePoolBuilder. This starts the needed sub-threads
which will run forever until we tell them to stop.
'''
with self._start_lock:
if self._running:
raise exceptions.BuilderError('Cannot start, already running.')
self._config = self._getAndValidateConfig()
self._running = True
# All worker threads share a single ZooKeeper instance/connection.
self.zk = zk.ZooKeeper()
self.zk.connect(self._config.zookeeper_servers.values())
self.log.debug('Starting listener for build jobs')
# Create build and upload worker objects
for i in range(self._num_builders):
w = BuildWorker(i, self._config_path, self.build_interval,
self.zk, self.dib_cmd)
w.start()
self._build_workers.append(w)
for i in range(self._num_uploaders):
w = UploadWorker(i, self._config_path, self.upload_interval,
self.zk)
w.start()
self._upload_workers.append(w)
if self.cleanup_interval > 0:
self._janitor = CleanupWorker(0, self._config_path,
self.cleanup_interval, self.zk)
self._janitor.start()
# Wait until all threads are running. Otherwise, we have a race
# on the worker _running attribute if shutdown() is called before
# run() actually begins.
workers = self._build_workers + self._upload_workers
if self._janitor:
workers += [self._janitor]
while not all([x.running for x in (workers)]):
time.sleep(0)
def _activate_virtualenv(self):
"""Run as a pre-exec function to activate current virtualenv
If we are invoked directly as /path/ENV/nodepool-builer (as
done by an init script, for example) then /path/ENV/bin will
not be in our $PATH, meaning we can't find disk-image-create.
Apart from that, dib also needs to run in an activated
virtualenv so it can find utils like dib-run-parts. Run this
before exec of dib to ensure the current virtualenv (if any)
is activated.
"""
if self._running_under_virtualenv():
activate_this = os.path.join(sys.prefix, "bin", "activate_this.py")
if not os.path.exists(activate_this):
raise exceptions.BuilderError("Running in a virtualenv, but "
"cannot find: %s" %
activate_this)
execfile(activate_this, dict(__file__=activate_this))
def runForever(self):
with self._start_lock:
if self._running:
raise exceptions.BuilderError('Cannot start, already running.')
self.load_config(self._config_path)
self._validate_config()
self.build_workers = []
self.upload_workers = []
for i in range(self._build_workers):
w = BuildWorker('Nodepool Builder Build Worker %s' % (i + 1, ),
builder=self)
self._initializeGearmanWorker(
w, self._config.gearman_servers.values())
self.build_workers.append(w)
for i in range(self._upload_workers):
w = UploadWorker('Nodepool Builder Upload Worker %s' %
(i + 1, ),
builder=self)
self._initializeGearmanWorker(
w, self._config.gearman_servers.values())
self.upload_workers.append(w)
self._registerGearmanFunctions(self._config.diskimages.values())
self._registerExistingImageUploads()
self.log.debug('Starting listener for build jobs')
self.threads = []
for worker in self.build_workers + self.upload_workers:
t = threading.Thread(target=worker.run)
t.daemon = True
t.start()
self.threads.append(t)
self._running = True
for t in self.threads:
t.join()
self._running = False
def to_path(self, images_dir, with_extension=True):
my_path = os.path.join(images_dir, self.image_id)
if with_extension:
if self.extension is None:
raise exceptions.BuilderError(
'Cannot specify image extension of None')
my_path += '.' + self.extension
md5_path = '%s.%s' % (my_path, 'md5')
md5 = self._checksum(md5_path)
if md5:
self.md5_file = md5_path
self.md5 = md5[0:32]
sha256_path = '%s.%s' % (my_path, 'sha256')
sha256 = self._checksum(sha256_path)
if sha256:
self.sha256_file = sha256_path
self.sha256 = sha256[0:64]
return my_path
def _buildImage(self, build_id, diskimage):
'''
Run the external command to build the diskimage.
:param str build_id: The ID for the build (used in image filename).
:param diskimage: The diskimage as retrieved from our config file.
:returns: An ImageBuild object of build-related data.
:raises: BuilderError if we failed to execute the build command.
'''
base = "-".join([diskimage.name, build_id])
image_file = DibImageFile(base)
filename = image_file.to_path(self._config.imagesdir, False)
env = os.environ.copy()
env['DIB_RELEASE'] = diskimage.release
env['DIB_IMAGE_NAME'] = diskimage.name
env['DIB_IMAGE_FILENAME'] = filename
# Note we use a reference to the nodepool config here so
# that whenever the config is updated we get up to date
# values in this thread.
if self._config.elementsdir:
env['ELEMENTS_PATH'] = self._config.elementsdir
# send additional env vars if needed
for k, v in diskimage.env_vars.items():
env[k] = v
img_elements = diskimage.elements
img_types = ",".join(diskimage.image_types)
qemu_img_options = ''
if 'qcow2' in img_types:
qemu_img_options = DEFAULT_QEMU_IMAGE_COMPAT_OPTIONS
cmd = ('%s -x -t %s --checksum --no-tmpfs %s -o %s %s' %
(self.dib_cmd, img_types, qemu_img_options, filename,
img_elements))
log = logging.getLogger("nodepool.image.build.%s" % (diskimage.name, ))
self.log.info('Running %s' % cmd)
try:
p = subprocess.Popen(shlex.split(cmd),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
preexec_fn=self._activate_virtualenv,
env=env)
except OSError as e:
raise exceptions.BuilderError("Failed to exec '%s'. Error: '%s'" %
(cmd, e.strerror))
while True:
ln = p.stdout.readline()
log.info(ln.strip())
if not ln:
break
p.wait()
# It's possible the connection to the ZK cluster could have been
# interrupted during the build. If so, wait for it to return.
# It could transition directly from SUSPENDED to CONNECTED, or go
# through the LOST state before CONNECTED.
while self._zk.suspended or self._zk.lost:
self.log.info("ZooKeeper suspended during build. Waiting")
time.sleep(SUSPEND_WAIT_TIME)
build_data = zk.ImageBuild()
build_data.builder = self._hostname
if self._zk.didLoseConnection:
self.log.info("ZooKeeper lost while building %s" % diskimage.name)
self._zk.resetLostFlag()
build_data.state = zk.FAILED
elif p.returncode:
self.log.info("DIB failed creating %s" % diskimage.name)
build_data.state = zk.FAILED
else:
self.log.info("DIB image %s is built" % diskimage.name)
build_data.state = zk.READY
build_data.formats = list(diskimage.image_types)
if self._statsd:
# record stats on the size of each image we create
for ext in img_types.split(','):
key = 'nodepool.dib_image_build.%s.%s.size' % (
diskimage.name, ext)
# A bit tricky because these image files may be sparse
# files; we only want the true size of the file for
# purposes of watching if we've added too much stuff
# into the image. Note that st_blocks is defined as
# 512-byte blocks by stat(2)
size = os.stat("%s.%s" % (filename, ext)).st_blocks * 512
self.log.debug("%s created image %s.%s (size: %d) " %
(diskimage.name, filename, ext, size))
self._statsd.gauge(key, size)
return build_data
def _runDibForImage(self, image, filename):
env = os.environ.copy()
env['DIB_RELEASE'] = image.release
env['DIB_IMAGE_NAME'] = image.name
env['DIB_IMAGE_FILENAME'] = filename
# Note we use a reference to the nodepool config here so
# that whenever the config is updated we get up to date
# values in this thread.
if self._config.elementsdir:
env['ELEMENTS_PATH'] = self._config.elementsdir
if self._config.scriptdir:
env['NODEPOOL_SCRIPTDIR'] = self._config.scriptdir
# send additional env vars if needed
for k, v in image.env_vars.items():
env[k] = v
img_elements = image.elements
img_types = ",".join(image.image_types)
qemu_img_options = ''
if 'qcow2' in img_types:
qemu_img_options = DEFAULT_QEMU_IMAGE_COMPAT_OPTIONS
if 'fake-' in image.name:
dib_cmd = 'nodepool/tests/fake-image-create'
else:
dib_cmd = 'disk-image-create'
cmd = ('%s -x -t %s --no-tmpfs %s -o %s %s' %
(dib_cmd, img_types, qemu_img_options, filename, img_elements))
log = logging.getLogger("nodepool.image.build.%s" %
(image.name,))
self.log.info('Running %s' % cmd)
try:
p = subprocess.Popen(
shlex.split(cmd),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
except OSError as e:
raise exceptions.BuilderError(
"Failed to exec '%s'. Error: '%s'" % (cmd, e.strerror)
)
while True:
ln = p.stdout.readline()
log.info(ln.strip())
if not ln:
break
p.wait()
ret = p.returncode
if ret:
raise exceptions.DibFailedError(
"DIB failed creating %s" % filename
)
def create_image(self, **kwargs):
raise exceptions.BuilderError("Test fail image upload.")
def _runDibForImage(self, image, filename):
env = os.environ.copy()
env['DIB_RELEASE'] = image.release
env['DIB_IMAGE_NAME'] = image.name
env['DIB_IMAGE_FILENAME'] = filename
# Note we use a reference to the nodepool config here so
# that whenever the config is updated we get up to date
# values in this thread.
if self._config.elementsdir:
env['ELEMENTS_PATH'] = self._config.elementsdir
if self._config.scriptdir:
env['NODEPOOL_SCRIPTDIR'] = self._config.scriptdir
# send additional env vars if needed
for k, v in image.env_vars.items():
env[k] = v
img_elements = image.elements
img_types = ",".join(image.image_types)
qemu_img_options = ''
if 'qcow2' in img_types:
qemu_img_options = DEFAULT_QEMU_IMAGE_COMPAT_OPTIONS
if 'fake-' in image.name:
dib_cmd = 'nodepool/tests/fake-image-create'
else:
dib_cmd = 'disk-image-create'
cmd = ('%s -x -t %s --no-tmpfs %s -o %s %s' %
(dib_cmd, img_types, qemu_img_options, filename, img_elements))
log = logging.getLogger("nodepool.image.build.%s" % (image.name, ))
self.log.info('Running %s' % cmd)
try:
p = subprocess.Popen(shlex.split(cmd),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
except OSError as e:
raise exceptions.BuilderError("Failed to exec '%s'. Error: '%s'" %
(cmd, e.strerror))
while True:
ln = p.stdout.readline()
log.info(ln.strip())
if not ln:
break
p.wait()
ret = p.returncode
if ret:
raise exceptions.DibFailedError("DIB failed creating %s" %
filename)
if self.statsd:
# record stats on the size of each image we create
for ext in img_types.split(','):
key = 'nodepool.dib_image_build.%s.%s.size' % (image.name, ext)
# A bit tricky because these image files may be sparse
# files; we only want the true size of the file for
# purposes of watching if we've added too much stuff
# into the image. Note that st_blocks is defined as
# 512-byte blocks by stat(2)
size = os.stat("%s.%s" % (filename, ext)).st_blocks * 512
self.log.debug("%s created image %s.%s (size: %d) " %
(image.name, filename, ext, size))
self.statsd.gauge(key, size)
