def testDeploySSHCluster(self):
options = VmManager.defaultRunOptions()
options.update({'username': ClusterTest.username,
'password': ClusterTest.password,
'mpi_machine_file': True,
'instanceType': self.instanceType,
'cluster_admin': 'root',
'cluster_user': '******',
'master_vmid': None,
'tolerate_failures': False,
'clean_after_failure': False,
'include_master': True,
'shared_folder': None,
'add_packages': None,
'ssh_hostbased': True,
'instanceNumber': self.instanceNumber,
'verboseLevel': 0,
'marketplaceEndpoint': 'https://marketplace.stratuslab.eu'})
configHolder = ConfigHolder(options)
runner = VmManagerFactory.create(self.image, configHolder)
cluster = Cluster(configHolder, runner, options['master_vmid'])
runner.runInstance()
self.assertTrue(cluster.deploy() in [0, 128])
runner.killInstances(runner.vmIds)
def _insert_required_run_option_defaults(self, holder, options=None):
defaults = VmManager.defaultRunOptions()
defaults['verboseLevel'] = -1
required_options = ['verboseLevel', 'vmTemplateFile',
'marketplaceEndpoint', 'vmRequirements',
'outVmIdsFile', 'inVmIdsFile', 'vncPort']
if options is not None:
for option in options:
if option not in required_options and option in defaults:
required_options.append(option)
for option in required_options:
if not holder.config.get(option):
holder.config[option] = defaults[option]
def _get_config_sizes(self):
"""
Create all of the node sizes based on the user configuration.
"""
size_map = {}
machine_types = VmManager.getDefaultInstanceTypes()
for name in machine_types.keys():
size = self._create_node_size(name, machine_types[name])
size_map[name] = size
machine_types = self.user_configurator.getUserDefinedInstanceTypes()
for name in machine_types.keys():
size = self._create_node_size(name, machine_types[name])
size_map[name] = size
return size_map.values()
def __init__(self, image, configHolder):
self.image = image
self.configHolder = configHolder
self.newImageGroupName = ''
self.newInstalledSoftwareName = ''
self.newInstalledSoftwareVersion = ''
self.newImageGroupVersion = ''
self.newImageGroupVersionWithManifestId = False
self.author = ''
self.title = ''
self.comment = ''
self.os = ''
self.authorEmail = ''
self.marketplaceEndpointNewimage = ''
self.endpoint = ''
self.extraOsReposUrls = ''
self.packages = ''
self.scripts = ''
self.prerecipe = ''
self.recipe = ''
self.verboseLevel = ''
self.shutdownVm = True
self.signManifest = True
self.vmStartTimeout = self.VM_START_TIMEOUT
self.vmPingTimeout = self.VM_PING_TIMEOUT
self.options = VmManager.defaultRunOptions()
self.options.update(configHolder.options)
self.configHolder.options.update(self.options)
configHolder.assign(self)
self._set_stdouterr()
credentials = AuthnFactory.getCredentials(self)
self.cloud = CloudConnectorFactory.getCloud(credentials)
self.cloud.setEndpoint(self.endpoint)
self.runner = None
self.vmAddress = None
self.vmId = None
self.vmIp = None
self.vmName = 'creator'
self.userPublicKeyFile = self.options.get('userPublicKeyFile',
Defaults.sshPublicKeyLocation)
self.userPrivateKeyFile = self.userPublicKeyFile.strip('.pub')
self.mainDisk = ''
self.extraDisk = ''
self.mountPointExtraDisk = '/media'
self.imageFile = ''
self.imageFileBundled = ''
self.excludeFromCreatedImage = \
self.excludeFromCreatedImageDefault + \
self.options.get('excludeFromCreatedImage', '').split(',')
self.installer = self.options.get('installer')
self.targetImageUri = ''
self.targetManifestUri = ''
self.manifest = ''
self.manifestObject = None
self.newManifestFileName = None
self.manifestLocalFileName = ''
self.__listener = CreatorBaseListener()
def parse(self):
default_options = VmManager.defaultRunOptions()
self.parser.usage = self.parser_usage
self.parser.description = self.parser_description
self.parser.add_option('-k', '--key', dest='userPublicKeyFile',
help='SSH public key(s) (.pub) to log on the machine. Default %s. In case of multiple keys, concatenate them to the file.' %
default_options['userPublicKeyFile'],
metavar='FILE',
default=default_options['userPublicKeyFile'])
self.parser.add_option('-t', '--type', dest='instanceType',
help='instance type to start (see --list-types for default)', metavar='TYPE',
default=VmManager.DEFAULT_INSTANCE_TYPE)
self.parser.add_option('--list-type', dest='listType',
help='list available instance types (deprecated)',
action='store_true')
self.parser.add_option('-l', '--list-types', dest='listType',
help='list available instance types',
default=False, action='store_true')
self.parser.add_option('--cpu', dest='vmCpu',
help='number of CPU cores',
default=None)
self.parser.add_option('--ram', dest='vmRam',
help='RAM in megabytes',
default=None)
self.parser.add_option('--swap', dest='vmSwap',
help='swap space in megabytes',
default=None)
self.parser.add_option('--context-file', dest='extraContextFile', metavar='FILE',
help='extra context file with one key=value per line',
default=default_options['extraContextFile'])
self.parser.add_option('--context', dest='extraContextData', metavar='CONTEXT',
help='extra context string (separate by %s)' % Util.cliLineSplitChar,
default=default_options['extraContextData'])
self.parser.add_option('--context-method', dest='defaultContextMethod', metavar='METHOD',
help='default contextualization method (one or cloud-init)',
default=default_options['defaultContextMethod'])
self.parser.add_option('--cloud-init', dest='cloudInit', metavar='PAIRS',
help='mimetype,file pairs (separate by %s)' % Util.cliLineSplitChar,
default=default_options['cloudInit'])
self.parser.add_option('--vnc-port', dest='vncPort', metavar='PORT', type='int',
help='VNC port number. Note for KVM it\'s the real one, not the '
'VNC port. So for VNC port 0 you should specify 5900, for '
'port 1 is 5901 and so on. ',
default=default_options['vncPort'])
self.parser.add_option('--vnc-listen', dest='vncListen', metavar='ADDRESS',
help='IP to listen on',
default=default_options['vncListen'])
self.parser.add_option('--keymap', dest='keymap', metavar='LANGUAGE',
help='Keymap to use.',
default=default_options['keymap'])
self.parser.add_option('--use-host-cpu', dest='useHostCpu',
help='Use host CPU extensions (flags)',
default=default_options['useHostCpu'],
action='store_true')
self.parser.add_option('--vm-requirements', dest='vmRequirements', metavar='REQUIREMENTS',
help='Advanced requirements for VM placement. '
'Available values depend on the cloud configuration. '
'e.g. --vm-requirements \'CLUSTER = \\"test\\" & MAGICNUMBER = 42\'',
default=None)
self.parser.add_option('--vm-network-model', dest='vmNetworkModel', metavar='MODEL',
help='NIC model. '
'Default: hypervisor default. Common value: virtio.',
default=None)
self.parser.add_option('--vm-template-file', dest='vmTemplateFile', metavar='FILE',
help='VM template file. Default %s' % default_options['vmTemplateFile'],
default=default_options['vmTemplateFile'])
self.parser.add_option('--vm-cpu-amount', dest='vmCpuAmount', metavar='CPU', type='float',
help='Percentage of CPU divided by 100 required for the Virtual Machine. '
'Half a processor is written 0.5. No default. If not provided, CPU value from '
'predefined instance types is used.',
default=default_options['vmCpuAmount'])
self.parser.add_option('--vm-disks-bus', dest='vmDisksBus', metavar='BUSTYPE',
help='VM disks bus type defined for all disks. Overrides "disks-bus" '
'element value defined in image manifest. '
'Available types: %s. ' % ', '.join(VmManager.DISKS_BUS_AVAILABLE) +
'If not provided, by default the value is taken from disks-bus '
'element of image manifest. If the latter is not set, '
'by default "%s" is assumed.' % VmManager.DISKS_BUS_DEFAULT,
default=default_options['vmDisksBus'])
MarketplaceUtil.addEndpointOption(self.parser)
AuthnCommand.addCloudEndpointOptions(self.parser)
super(Runnable, self).parse()
options, self.args = self.parser.parse_args()
self._assignOptions(default_options, options)
def __init__(self, image, configHolder):
self.image = image
self.configHolder = configHolder
self.newImageGroupName = ""
self.newInstalledSoftwareName = ""
self.newInstalledSoftwareVersion = ""
self.newImageGroupVersion = ""
self.newImageGroupVersionWithManifestId = False
self.author = ""
self.title = ""
self.comment = ""
self.os = ""
self.authorEmail = ""
self.marketplaceEndpointNewimage = ""
self.endpoint = ""
self.extraOsReposUrls = ""
self.packages = ""
self.scripts = ""
self.prerecipe = ""
self.recipe = ""
self.verboseLevel = ""
self.shutdownVm = True
self.signManifest = True
self.vmStartTimeout = self.VM_START_TIMEOUT
self.vmPingTimeout = self.VM_PING_TIMEOUT
self.options = VmManager.defaultRunOptions()
self.options.update(configHolder.options)
self.configHolder.options.update(self.options)
configHolder.assign(self)
dateNow = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
self.stdout = open("/tmp/stratuslab_%s.log" % dateNow, "a")
self.stderr = open("/tmp/stratuslab_%s.err" % dateNow, "a")
credentials = AuthnFactory.getCredentials(self)
self.cloud = CloudConnectorFactory.getCloud(credentials)
self.cloud.setEndpoint(self.endpoint)
self.runner = None
self.vmAddress = None
self.vmId = None
self.vmIp = None
self.vmName = "creator"
self.userPublicKeyFile = self.options.get("userPublicKeyFile", Defaults.sshPublicKeyLocation)
self.userPrivateKeyFile = self.userPublicKeyFile.strip(".pub")
self.mainDisk = ""
self.extraDisk = ""
self.mountPointExtraDisk = "/media"
self.imageFile = ""
self.imageFileBundled = ""
self.excludeFromCreatedImage = self.excludeFromCreatedImageDefault + self.options.get(
"excludeFromCreatedImage", ""
).split(",")
self.installer = self.options.get("installer")
self.targetImageUri = ""
self.targetManifestUri = ""
self.manifest = ""
self.manifestObject = None
self.newManifestFileName = None
self.manifestLocalFileName = ""
self.__listener = CreatorBaseListener()
def parse(self):
options = VmManager.defaultRunOptions()
self.parser.add_option('-n', '--number', dest='instanceNumber',
help='number of machines to start (default 1)', metavar='INT',
default=options['instanceNumber'], type='int')
self.parser.add_option('--local-ip', dest='isLocalIp', action='store_true',
help='By default, virtual machines are assigned a public IP address. '
'With this option, a local address is provided, which means that it '
'not be reachable from the outside, but will be reachable from other '
'instances in the same data-center.',
default=False)
self.parser.add_option('--private-ip', dest='isPrivateIp', action='store_true',
help='By default, virtual machines are assigned a public IP address. '
'With this option, a private address is provided, which means that it '
'not be reachable anywhere. This is useful for instances that initiate '
'the connection to other machinces.',
default=False)
self.parser.add_option('--save', dest='saveDisk', action='store_true',
help='save image after VM shutdown',
default=False)
self.parser.add_option('--author-email', dest='authorEmail',
help='Used with --save option. Email address of the author of '
'the new image. After image is saved a notification is sent '
'with further instructions.',
default='', metavar='EMAIL')
self.parser.add_option('--author', dest='author',
help='Used with --save option. Author of the new image',
default='', metavar='NAME')
self.parser.add_option('--image-version', dest='newImageGroupVersion',
help='Used with --save option. Version for the new image.',
default='', metavar='VERSION')
self.parser.add_option('--title', dest='title',
help='Used with --save option. Title of the new image',
default='', metavar='TEXT')
self.parser.add_option('--comment', dest='comment',
help='Used with --save option. Description of the new image',
default='', metavar='TEXT')
self.parser.add_option('--marketplace-endpoint-newimage', dest='marketplaceEndpointNewimage',
help='Used with --save option. Marketplace to register the new image manifest in. '
'No default. If not provided, either base image Marketplace or defined '
'by cloud site will be used.',
default='', metavar='URL')
self.parser.add_option('-o', '--output', dest='outVmIdsFile',
help='save vm-id to a file', metavar='FILE',
default=None)
self.parser.add_option('--volatile-disk', dest='extraDiskSize',
help='volatile data disk size in GB', metavar='INT',
action='callback', callback=self.diskSizeOptionCallback,
default=0, type='int')
self.parser.add_option('--persistent-disk', dest='persistentDiskUUID',
help='persistent disk UUID',
default=None)
self.parser.add_option('--readonly-disk', dest='readonlyDiskId',
help='marketplace readonly disk image ID',
default=None, metavar='MARKETPLACEID')
self.parser.add_option('--qcow-disk-format', dest='useQcowDiskFormat',
help='launch instance from an image in qcow2 disk format. Only '
'used when image is provided as URL. When image is given by ID, disk '
'format is inferred from the image metadata.',
action='store_true',
default=False)
self.parser.add_option('--address', dest='specificAddressRequest',
help='request a specific ip address',
metavar='IP', default=None)
self.parser.add_option('--no-check-image-url', dest='noCheckImageUrl',
help='Turn off image availability check. '
'Do not check if the location URLs in the image metadata exist.',
action='store_true',
default=False)
self.parser.add_option('--vm-name', dest='vmName',
help='name of VM. If not given, a default name will be assigned by cloud layer.',
metavar='NAME', default=options['vmName'])
self.parser.add_option('--notify', dest='msgRecipients', action='append',
help='recipient coordinates (host,vhost,user,pass,queue)',
default=[])
StorageCommand.addPDiskEndpointOptions(self.parser)
super(MainProgram, self).parse()
def parse(self):
options = VmManager.defaultRunOptions()
self.parser.add_option('--shared-folder', dest='shared_folder', action='store',
help='folder to share over NFS',
default=None, type='string')
self.parser.add_option('--ssh-hostbased', dest='ssh_hostbased', action='store_true',
help='Enable cluster integration through hostbased ssh authentication',
default=False)
self.parser.add_option('--cluster-admin', dest='cluster_admin', action='store',
help='Username for cluster administrator (default root)',
default='root', type='string')
self.parser.add_option('--cluster-user', dest='cluster_user', action='store',
help='Additional username to be created',
default=None, type='string')
self.parser.add_option('--mpi-machine-file', dest='mpi_machine_file', action='store_true',
help='Defines that a machinefile should be prepared for MPI, listing all available '
'workers and their respective number of cores (slot value). (Default=False)',
default=False)
self.parser.add_option('--include-master', dest='include_master', action='store_true',
help='Defines that the master node should be considered '
'as part of the worker nodes. (Default=False)',
default=False)
self.parser.add_option('--add-packages', dest='add_packages', action='store',
help='Provide comma separated list of additional packages that you wish to install',
default=None, type='string')
self.parser.add_option('--master-vmid', dest='master_vmid', action='store',
help='VMID of master node if this has been instantiated separately',
default=None, type='int')
self.parser.add_option('--tolerate-failures', dest='tolerate_failures', action='store_true',
help='If set true the program will try to configure the '
'cluster with as many nodes as it can '
'in case some of them fail during instantiation',
default=False)
self.parser.add_option('--clean-after-failure', dest='clean_after_failure', action='store_true',
help='In case the program stops on failure it will try to kill all VMs instantiated',
default=True)
self.parser.add_option('-n', '--number', dest='instanceNumber',
help='number of machines to start (default 1) for the cluster', metavar='INT',
default=options['instanceNumber'], type='int')
self.parser.add_option('--save', dest='saveDisk', action='store_true',
help='save image after VM shutdown',
default=False)
self.parser.add_option('-o', '--output', dest='outVmIdsFile',
help='save vm-id to a file', metavar='FILE',
default=None)
self.parser.add_option('--volatile-disk', dest='extraDiskSize',
help='volatile data disk size in GB', metavar='INT',
action='callback', callback=self.diskSizeOptionCallback,
default=0, type='int')
self.parser.add_option('--persistent-disk', dest='persistentDiskUUID',
help='persistent disk UUID',
default=None)
self.parser.add_option('--readonly-disk', dest='readonlyDiskId',
help='marketplace readonly disk image ID',
default=None, metavar='MARKETPLACEID')
self.parser.add_option('--qcow-disk-format', dest='useQcowDiskFormat',
help='launch instance from an image in qcow2 disk format. This option requires ',
action='store_true',
default=False)
self.parser.add_option('--address', dest='specificAddressRequest',
help='request a specific ip address',
metavar='IP', default=None)
self.parser.add_option('--no-check-image-url', dest='noCheckImageUrl',
help='Do not check image availability.',
action='store_true',
default=False)
self.parser.add_option('--vm-name', dest='vmName',
help='name of VM. If not given, a default name will be assigned by cloud layer.',
metavar='NAME', default=options['vmName'])
StorageCommand.addPDiskEndpointOptions(self.parser)
super(MainProgram, self).parse()
def deploy_node(self, **kwargs):
"""
Create a new node, and start deployment.
@keyword deploy: Deployment to run once machine is online and
availble to SSH.
@type deploy: L{Deployment}
@keyword ssh_username: Optional name of the account which is used
when connecting to
SSH server (default is root)
@type ssh_username: C{str}
@keyword ssh_alternate_usernames: Optional list of ssh usernames to
try to connect with if using the
default one fails
@type ssh_alternate_usernames: C{list}
@keyword ssh_port: Optional SSH server port (default is 22)
@type ssh_port: C{int}
@keyword ssh_timeout: Optional SSH connection timeout in seconds
(default is 10)
@type ssh_timeout: C{float}
@keyword auth: Initial authentication information for the node
(optional)
@type auth: L{NodeAuthSSHKey} or L{NodeAuthPassword}
@keyword ssh_key: A path (or paths) to an SSH private key with which
to attempt to authenticate. (optional)
@type ssh_key: C{str} or C{list} of C{str}
@keyword timeout: How many seconds to wait before timing out.
(default is 600)
@type timeout: C{int}
@keyword max_tries: How many times to retry if a deployment fails
before giving up (default is 3)
@type max_tries: C{int}
@keyword ssh_interface: The interface to wait for. Default is
'public_ips', other option is 'private_ips'.
@type ssh_interface: C{str}
@return: The newly created and deployed node.
@rtype: L{StratusLabNode}
@inherits: L{NodeDriver.deploy_node}
"""
if 'ssh_key' not in kwargs:
location = kwargs.get('location', self.default_location)
holder = self._get_config_section(location)
if hasattr(holder, 'userPrivateKeyFile'):
kwargs['ssh_key'] = getattr(holder, 'userPrivateKeyFile')
else:
defaults = VmManager.defaultRunOptions()
if 'userPrivateKeyFile' in defaults:
kwargs['ssh_key'] = defaults.get('userPrivateKeyFile')
return super(StratusLabNodeDriver, self).deploy_node(**kwargs)
