def _load_file(conf_file):
'''Parse a configuration file and return it as a dictionary. The option
values are checked for type correctness against a default Config object.
:param conf_file: file path (string) or file handler to the configuration
file in YAML syntax
:return: dictionary parsed from the configuration file
:raise CheckbConfigError: if any problem occurs during the parsing or
some values have incorrect variable type
'''
# convert file path to file handle if needed
if isinstance(conf_file, basestring):
try:
conf_file = open(conf_file)
except IOError as e:
log.exception('Could not open config file: %s', conf_file)
raise exc.CheckbConfigError(e)
filename = (conf_file.name
if hasattr(conf_file, 'name') else '<unnamed file>')
try:
conf_obj = yaml.safe_load(conf_file)
except yaml.YAMLError as e:
log.exception('Could not parse config file: %s', filename)
raise exc.CheckbConfigError(e)
# config file might be empty (all commented out), returning None. For
# further processing, let's replace it with empty dict
if conf_obj is None:
conf_obj = {}
# check correct types
# we should receive a single dictionary with keyvals
if not isinstance(conf_obj, abc.Mapping):
raise exc.CheckbConfigError(
'The config file %s does not have '
'a valid structure. Instead of a mapping, it is recognized as: %s'
% (filename, type(conf_obj)))
default_conf = Config()
for option, value in conf_obj.items():
# check for unknown options
try:
default_value = getattr(default_conf, option)
except AttributeError:
log.warning('Unknown option "%s" in the config file %s', option,
filename)
continue
# check for correct type
assert default_value is not None, \
"Default values must not be None: %s" % option
if type(default_value) is not type(value):
raise exc.CheckbConfigError(
'Option "%s" in config file %s '
'has an invalid type. Expected: %s, Found: %s' %
(option, filename, type(default_value), type(value)))
return conf_obj
def git_origin_url(taskdir):
try:
gitconfig_filename = os.path.join(taskdir, '.git/config')
gitconfig = configparser.ConfigParser()
gitconfig.read(gitconfig_filename)
task_repo_url = gitconfig['remote "origin"']['url']
except TypeError as e:
log.exception(e)
task_repo_url = None
return task_repo_url
def teardown(self):
'''Tear down the virtual machine by stopping it and removing it from the host machine.
:raises CheckbRemoteError: if there is a failure while stopping or removing the virtual
machine instance
'''
tc_instance = self._check_existing_instance(should_exist=True)
try:
tc_instance.remove(autostop=True)
except TestcloudInstanceError as e:
log.exception("Error while tearing down instance {}".format(
self.instancename))
raise exc.CheckbRemoteError(e)
def _prepare_image(self, distro, release, flavor, arch):
'''Use testcloud to prepare an image for local booting
:param str distro: Distro to use in image discovery
:param str release: Distro's release to use in image discovery
:param str flavor: base-image flavor to use in image discovery
:param str arch: arch to use in image discovery
:raises CheckbImageNotFoundError: when base image of the required type is not found
:raises CheckbImageError: for errors in preparing the image with testcloud
'''
tc_image = None
try:
if config.get_config().force_imageurl:
img_url = config.get_config().imageurl
else:
log.debug(
"Looking for image with DISTRO: %s, RELEASE: %s, FLAVOR: %s, ARCH: %s"
% (distro, release, flavor, arch))
img_url = ImageFinder.get_latest(distro=distro,
release=release,
flavor=flavor,
arch=arch)
except exc.CheckbImageNotFoundError as e:
log.error(e)
raise
log.debug("Preparing image {} for task {}".format(img_url, self.uuid))
try:
tc_image = image.Image(img_url)
# symlink the image instead of copying it to the testcloud dir, because our user only
# expects image handling in checkb dirs, and we remove all minion instances
# immediately after task execution anyway
tc_image.prepare(copy=False)
except TestcloudImageError as e:
log.exception(e)
raise exc.CheckbImageError(
"There was an error while preparing the "
"testcloud image", e)
return tc_image
def download(url, dirname, filename=None, cachedir=None):
'''Download a file.
:param str url: file URL to download
:param str dirname: directory path; if the directory does not exist, it gets
created (and all its parent directories).
:param str filename: name of downloaded file; if not provided, the basename
is extracted from URL
:param str cachedir: If set, the file will be downloaded to a cache
directory specified by this parameter. If the file is
already present and of the same length, download is skipped.
The requested destination file (``dirname/filename``)
will be a symlink to the cached file.
This directory is automatically created if not present.
:return: the path to the downloaded file
:rtype: str
:raise CheckbRemoteError: if download fails
'''
if not filename:
filename = os.path.basename(url)
dl_dest = dest = os.path.abspath(os.path.join(dirname, filename))
dirs_to_create = [dirname]
if cachedir:
dl_dest = os.path.join(cachedir, filename)
dirs_to_create.append(cachedir)
for directory in dirs_to_create:
makedirs(directory)
# check file existence and validity
download = True
if os.path.exists(dl_dest):
if _same_length(dl_dest, url):
log.debug('Already downloaded: %s', dl_dest)
download = False
else:
log.debug(
'Cached file %s differs from its online version. '
'Redownloading.', dl_dest)
# download the file
if download:
log.debug('Downloading%s: %s', ' (cached)' if cachedir else '', url)
try:
_download(url, dl_dest)
except requests.exceptions.RequestException as e:
log.debug('Download failed: %s', e)
# the file can be incomplete, remove
if os.path.exists(dl_dest):
try:
os.remove(dl_dest)
except OSError:
log.warning('Could not delete incomplete file: %s',
dl_dest)
raise CheckbRemoteError(e, errno=e.response.status_code)
# create a symlink if the download was cached
if cachedir:
try:
if os.path.exists(dest):
# if there already is something at the destination, we need to
# remove it first
os.remove(dest)
os.symlink(dl_dest, dest)
except OSError:
log.exception("Can't create symlink %s -> %s", dl_dest, dest)
raise
return dest