def boot_manifest(manifest_data, boot_vars={}):
from bootstrapvz.common.tools import load_data
build_servers = load_data('build-servers.yml')
from bootstrapvz.remote.build_servers import pick_build_server
build_server = pick_build_server(build_servers, manifest_data)
manifest_data = build_server.apply_build_settings(manifest_data)
from bootstrapvz.base.manifest import Manifest
manifest = Manifest(data=manifest_data)
import importlib
provider_module = importlib.import_module('tests.system.providers.' +
manifest.provider['name'])
prepare_bootstrap = getattr(provider_module, 'prepare_bootstrap', noop)
with prepare_bootstrap(manifest, build_server):
bootstrap_info = None
log.info('Connecting to build server')
with build_server.connect() as connection:
log.info('Building manifest')
bootstrap_info = connection.run(manifest)
log.info('Creating and booting instance')
with provider_module.boot_image(manifest, build_server, bootstrap_info,
**boot_vars) as instance:
yield instance
def pick_build_server(build_servers, manifest, preferences={}):
# Validate the build servers list
from bootstrapvz.common.tools import load_data, rel_path
schema = load_data(rel_path(__file__, 'build-servers-schema.yml'))
import jsonschema
jsonschema.validate(build_servers, schema)
if manifest['provider']['name'] == 'ec2':
must_bootstrap = 'ec2-' + manifest['volume']['backing']
else:
must_bootstrap = manifest['provider']['name']
def matches(name, settings):
if preferences.get('name', name) != name:
return False
if preferences.get('release', settings['release']) != settings['release']:
return False
if must_bootstrap not in settings['can_bootstrap']:
return False
return True
for name, settings in build_servers.iteritems():
if not matches(name, settings):
continue
if settings['type'] == 'local': # pylint: disable=no-else-return
from .local import LocalBuildServer
return LocalBuildServer(name, settings)
else:
from .remote import RemoteBuildServer
return RemoteBuildServer(name, settings)
raise Exception('Unable to find a build server that matches your preferences.')
def pick_build_server(build_servers, manifest, preferences={}):
# Validate the build servers list
from bootstrapvz.common.tools import load_data
import os.path
schema = load_data(os.path.normpath(os.path.join(os.path.dirname(__file__), 'build-servers-schema.yml')))
import jsonschema
jsonschema.validate(build_servers, schema)
if manifest['provider']['name'] == 'ec2':
must_bootstrap = 'ec2-' + manifest['volume']['backing']
else:
must_bootstrap = manifest['provider']['name']
def matches(name, settings):
if preferences.get('name', name) != name:
return False
if preferences.get('release', settings['release']) != settings['release']:
return False
if must_bootstrap not in settings['can_bootstrap']:
return False
return True
for name, settings in build_servers.iteritems():
if not matches(name, settings):
continue
if settings['type'] == 'local':
from local import LocalBuildServer
return LocalBuildServer(name, settings)
else:
from remote import RemoteBuildServer
return RemoteBuildServer(name, settings)
raise Exception('Unable to find a build server that matches your preferences.')
def main():
"""Main function for invoking the bootstrap process remotely
"""
# Get the commandline arguments
opts = get_opts()
from bootstrapvz.common.tools import load_data
# load the manifest data, we might want to modify it later on
manifest_data = load_data(opts['MANIFEST'])
# load the build servers file
build_servers = load_data(opts['--servers'])
# Pick a build server
from build_servers import pick_build_server
preferences = {}
if opts['--name'] is not None:
preferences['name'] = opts['--name']
if opts['--release'] is not None:
preferences['release'] = opts['--release']
build_server = pick_build_server(build_servers, manifest_data, preferences)
# Apply the build server settings to the manifest (e.g. the virtualbox guest additions path)
manifest_data = build_server.apply_build_settings(manifest_data)
# Load the manifest
from bootstrapvz.base.manifest import Manifest
manifest = Manifest(path=opts['MANIFEST'], data=manifest_data)
# Set up logging
from bootstrapvz.base.main import setup_loggers
setup_loggers(opts)
# Register deserialization handlers for objects
# that will pass between server and client
from . import register_deserialization_handlers
register_deserialization_handlers()
# Everything has been set up, connect to the server and begin the bootstrapping process
with build_server.connect() as connection:
connection.run(manifest,
debug=opts['--debug'],
dry_run=opts['--dry-run'])
def main():
"""Main function for invoking the bootstrap process remotely
"""
# Get the commandline arguments
opts = get_opts()
from bootstrapvz.common.tools import load_data
# load the manifest data, we might want to modify it later on
manifest_data = load_data(opts['MANIFEST'])
# load the build servers file
build_servers = load_data(opts['--servers'])
# Pick a build server
from build_servers import pick_build_server
preferences = {}
if opts['--name'] is not None:
preferences['name'] = opts['--name']
if opts['--release'] is not None:
preferences['release'] = opts['--release']
build_server = pick_build_server(build_servers, manifest_data, preferences)
# Apply the build server settings to the manifest (e.g. the virtualbox guest additions path)
manifest_data = build_server.apply_build_settings(manifest_data)
# Load the manifest
from bootstrapvz.base.manifest import Manifest
manifest = Manifest(path=opts['MANIFEST'], data=manifest_data)
# Set up logging
from bootstrapvz.base.main import setup_loggers
setup_loggers(opts)
# Register deserialization handlers for objects
# that will pass between server and client
from . import register_deserialization_handlers
register_deserialization_handlers()
# Everything has been set up, connect to the server and begin the bootstrapping process
with build_server.connect() as connection:
connection.run(manifest,
debug=opts['--debug'],
dry_run=opts['--dry-run'])
def load_data(self, data=None): """Loads the manifest and performs a basic validation. This function reads the manifest and performs some basic validation of the manifest itself to ensure that the properties required for initalization are accessible (otherwise the user would be presented with some cryptic error messages). """ if data is None: self.data = load_data(self.path) else: self.data = data from . import validate_manifest # Validate the manifest with the base validation function in __init__ validate_manifest(self.data, self.schema_validator, self.validation_error)
def schema_validator(self, data, schema_path): """This convenience function is passed around to all the validation functions so that they may run a json-schema validation by giving it the data and a path to the schema. :param dict data: Data to validate (normally the manifest data) :param str schema_path: Path to the json-schema to use for validation """ import jsonschema schema = load_data(schema_path) try: jsonschema.validate(data, schema) except jsonschema.ValidationError as e: self.validation_error(e.message, e.path)
def schema_validator(self, data, schema_path):
"""This convenience function is passed around to all the validation functions
so that they may run a json-schema validation by giving it the data and a path to the schema.
:param dict data: Data to validate (normally the manifest data)
:param str schema_path: Path to the json-schema to use for validation
"""
import jsonschema
schema = load_data(schema_path)
try:
jsonschema.validate(data, schema)
except jsonschema.ValidationError as e:
self.validation_error(e.message, e.path)
def load_data(self, data=None):
"""Loads the manifest and performs a basic validation.
This function reads the manifest and performs some basic validation of
the manifest itself to ensure that the properties required for initalization are accessible
(otherwise the user would be presented with some cryptic error messages).
"""
if data is None:
self.data = load_data(self.path)
else:
self.data = data
from . import validate_manifest
# Validate the manifest with the base validation function in __init__
validate_manifest(self.data, self.schema_validator, self.validation_error)
def __init__(self, path=None, data=None):
"""Initializer: Given a path we load, validate and parse the manifest.
To create the manifest from dynamic data instead of the contents of a file,
provide a properly constructed dict as the data argument.
:param str path: The path to the manifest (ignored, when `data' is provided)
:param str data: The manifest data, if it is not None, it will be used instead of the contents of `path'
"""
if path is None and data is None:
raise ManifestError('`path\' or `data\' must be provided')
self.path = path
self.metaschema = load_data(rel_path(__file__, 'metaschema.json'))
self.load_data(data)
self.load_modules()
self.validate()
self.parse()
def boot_manifest(manifest_data, boot_vars={}):
from bootstrapvz.common.tools import load_data
build_servers = load_data('build-servers.yml')
from bootstrapvz.remote.build_servers import pick_build_server
build_server = pick_build_server(build_servers, manifest_data)
manifest_data = build_server.apply_build_settings(manifest_data)
from bootstrapvz.base.manifest import Manifest
manifest = Manifest(data=manifest_data)
import importlib
provider_module = importlib.import_module('tests.integration.providers.' + manifest.provider['name'])
prepare_bootstrap = getattr(provider_module, 'prepare_bootstrap', noop)
with prepare_bootstrap(manifest, build_server):
bootstrap_info = None
log.info('Connecting to build server')
with build_server.connect() as connection:
log.info('Building manifest')
bootstrap_info = connection.run(manifest)
log.info('Creating and booting instance')
with provider_module.boot_image(manifest, build_server, bootstrap_info, **boot_vars) as instance:
yield instance
import os.path
import glob
import random
import string
from bootstrapvz.common.tools import load_data
partial_json = glob.glob(os.path.join(os.path.dirname(__file__), '*.yml'))
partial_yaml = glob.glob(os.path.join(os.path.dirname(__file__), '*.json'))
partials = {}
for path in partial_json + partial_yaml:
key = os.path.splitext(os.path.basename(path))[0]
if key in partials:
msg = 'Error when loading partial manifests: The partial {key} exists twice'.format(key=key)
raise Exception(msg)
partials[key] = load_data(path)
pool = string.ascii_uppercase + string.ascii_lowercase + string.digits
random_password = ''.join(random.choice(pool) for _ in range(16))
partials['root_password']['plugins']['root_password']['password'] = random_password
def merge_manifest_data(standard_partials=[], custom=[]):
import yaml
manifest_data = [partials[name] for name in standard_partials]
manifest_data.extend(yaml.load(data) for data in custom)
return merge_dicts(*manifest_data)
# Snatched from here: http://stackoverflow.com/a/7205107
def merge_dicts(*args):
import os.path
import glob
import random
import string
from bootstrapvz.common.tools import load_data
partial_json = glob.glob(os.path.join(os.path.dirname(__file__), '*.yml'))
partial_yaml = glob.glob(os.path.join(os.path.dirname(__file__), '*.json'))
partials = {}
for path in partial_json + partial_yaml:
key = os.path.splitext(os.path.basename(path))[0]
if key in partials:
msg = 'Error when loading partial manifests: The partial {key} exists twice'.format(key=key)
raise Exception(msg)
partials[key] = load_data(path)
pool = string.ascii_uppercase + string.ascii_lowercase + string.digits
random_password = ''.join(random.choice(pool) for _ in range(16))
partials['root_password']['plugins']['root_password']['password'] = random_password
def merge_manifest_data(standard_partials=[], custom=[]):
import yaml
manifest_data = [partials[name] for name in standard_partials]
manifest_data.extend(yaml.load(data) for data in custom)
return merge_dicts(*manifest_data)
# Snatched from here: http://stackoverflow.com/a/7205107
def merge_dicts(*args):