def vm_create(self, vm_name, vm_type = "CernVM", vm_user = "******", vm_networkassoc = "",
vm_image = "", vm_mem = 1, vm_cores = 1, vm_storage = 30, customization = None, vm_keepalive = 0,
job_proxy_file_path = None, myproxy_creds_name = None, myproxy_server = None,
myproxy_server_port = None, job_per_core = False, proxy_non_boot = False,
vmimage_proxy_file = None, vmimage_proxy_file_path = None, vm_loc = ''):
log.debug("Running new instance with Marketplace id %s in StratusLab" % str(vm_loc))
runner = None
if vm_loc not in StratusLabCluster.__idMap:
runner = Runner(vm_loc, StratusLabCluster._v_configHolder) #vm_loc: URL of VM or key? Does not seem to matter in Runner (l. 506)
StratusLabCluster.__idMap[vm_loc] = runner
else:
runner = StratusLabCluster.__idMap[vm_loc]
try:
ids = runner.runInstance()
log.debug("Created instances: %s" % str(ids))
#for new_id in ids:
new_id = ids[len(ids) - 1]
if not vm_keepalive and self.keep_alive: #if job didn't set a keep_alive use the clouds default
vm_keepalive = self.keep_alive
new_vm = cluster_tools.VM(name = vm_name, id = str(new_id), vmtype = vm_type, user = vm_user,
network = vm_networkassoc, image = vm_image, memory = vm_mem,
cpucores = vm_cores, storage = vm_storage, keep_alive = vm_keepalive,
myproxy_creds_name = myproxy_creds_name, myproxy_server = myproxy_server,
myproxy_server_port = myproxy_server_port, job_per_core = job_per_core)
StratusLabCluster.__vmMap[str(new_id)] = vm_loc
if vm_loc not in self.__runnerIds:
self.__runnerIds[vm_loc] = [str(new_id),]
else:
self.__runnerIds[vm_loc].append(str(new_id))
self.vms.append(new_vm)
try:
self.resource_checkout(new_vm)
except:
log.exception("Unexpected error checking out resources when creating a VM. Programming error?")
return self.ERROR
#endfor
return 0
except Exception, e:
log.debug("Exception running new instance in StratusLab: %s" %str(e))
#import traceback
#traceback.print_exc()
return -1
def _insert_required_run_option_defaults(self, holder):
defaults = Runner.defaultRunOptions()
defaults['verboseLevel'] = -1
required_options = ['verboseLevel', 'vmTemplateFile',
'marketplaceEndpoint', 'vmRequirements',
'outVmIdsFile', 'inVmIdsFile']
for option in required_options:
if not holder.config.get(option):
holder.config[option] = defaults[option]
def _insert_required_run_option_defaults(self, holder):
defaults = Runner.defaultRunOptions()
defaults['verboseLevel'] = -1
required_options = [
'verboseLevel', 'vmTemplateFile', 'marketplaceEndpoint',
'vmRequirements', 'outVmIdsFile', 'inVmIdsFile'
]
for option in required_options:
if not holder.config.get(option):
holder.config[option] = defaults[option]
def __setRunnerIds(self, runnerIds):
self.__runnerIds = runnerIds
for key in self.__runnerIds.keys():
if len(self.__runnerIds[key]) > 0:
run = Runner(key, StratusLabCluster.__configHolder)
self.__idMap[key] = run
for runner_id in self.__runnerIds[key]:
self.__vmMap[runner_id] = str(key)
run.vmIds.append(runner_id)
else:
del self.__runnerIds[key]
def _get_config_sizes(self):
"""
Create all of the node sizes based on the user configuration.
"""
size_map = {}
machine_types = Runner.getDefaultInstanceTypes()
for name in machine_types.keys():
size = self._create_node_size(name, machine_types[name])
size_map[name] = size
machine_types = self.user_configurator.getUserDefinedInstanceTypes()
for name in machine_types.keys():
size = self._create_node_size(name, machine_types[name])
size_map[name] = size
return size_map.values()
def _get_config_sizes(self):
"""
Create all of the node sizes based on the user configuration.
"""
size_map = {}
machine_types = Runner.getDefaultInstanceTypes()
for name in machine_types.keys():
size = self._create_node_size(name, machine_types[name])
size_map[name] = size
machine_types = self.user_configurator.getUserDefinedInstanceTypes()
for name in machine_types.keys():
size = self._create_node_size(name, machine_types[name])
size_map[name] = size
return size_map.values()
def _create_runner(self, name, size, image, location=None, auth=None):
location = location or self.default_location
holder = self._get_config_section(location)
self._insert_required_run_option_defaults(holder)
holder.set('vmName', name)
pubkey_file = None
if isinstance(auth, NodeAuthSSHKey):
_, pubkey_file = tempfile.mkstemp(suffix='_pub.key', prefix='ssh_')
with open(pubkey_file, 'w') as f:
f.write(auth.pubkey)
holder.set('userPublicKeyFile', pubkey_file)
# The cpu attribute is only included in the StratusLab
# subclass of NodeSize. Recover if the user passed in a
# normal NodeSize; default to 1 CPU in this case.
try:
cpu = size.cpu
except AttributeError:
cpu = 1
holder.set('vmCpu', cpu)
holder.set('vmRam', size.ram)
holder.set('vmSwap', size.disk)
runner = Runner(image.id, holder)
if pubkey_file:
os.remove(pubkey_file)
return runner
class StratusLabCluster(cluster_tools.ICluster):
"""
Connector class for StratusLab Clouds - Has not been tested for years.
"""
VM_TARGETSTATE = "Running"
VM_NODES = "1"
VM_SHUTDOWN = 90.0
ERROR = 1
VM_STATES = {
'INIT': 'Starting',
'BOOT': 'Starting',
'PROLOG': 'Starting',
'PENDING': 'Starting',
'HOLD': 'Starting',
'RUNNING': 'Running',
'ACTIVE': 'Running',
'STOPPED': 'Running',
'SUSPENDED': 'Running',
'DONE': 'Shutdown',
'EPILOG': 'Shutdown',
'FAILED': 'Error',
'FAILURE': 'Error',
'UNKNOWN': 'Error',
}
log = utilities.get_cloudscheduler_logger()
config = ConfigParser.ConfigParser()
config_file = os.path.expanduser('~/.stratuslab/stratuslab-user.cfg')
config_section = 'default'
config.read(config_file)
#Read the stratuslab configuration file
username, password, endpoint = None, None, None
try:
username = config.get(config_section, 'username')
except ConfigParser.NoOptionError:
log.error(
"Stratuslab config file %s does not contain username. VM creation will fail",
config_file)
except ConfigParser.NoSectionError:
log.error(
"No section '%s' found Stratuslab config file '%s'. \
Do the file and section exist? VM creation will fail",
config_section, config_file)
try:
password = config.get(config_section, 'password')
except ConfigParser.NoOptionError:
log.error(
"Stratuslab config file %s does not contain password. VM creation will fail",
config_file)
except ConfigParser.NoSectionError:
log.error(
"No section '%s' found Stratuslab config file '%s'. \
Do the file and section exist? VM creation will fail",
config_section, config_file)
try:
endpoint = config.get(config_section, 'endpoint')
except ConfigParser.NoOptionError:
log.error(
"Stratuslab config file %s does not contain endpoint. VM creation will fail",
config_file)
except ConfigParser.NoSectionError:
log.error(
"No section '%s' found Stratuslab config file '%s'. \
Do the file and section exist? VM creation will fail",
config_section, config_file)
options = Runner.defaultRunOptions()
options.update({
'username': username,
'password': password,
'endpoint': endpoint,
'marketplaceEndpoint': 'https://marketplace.stratuslab.eu',
'mpi_machine_file': True,
'cluster_admin': 'root',
'master_vmid': None,
'tolerate_failures': False,
'clean_after_failure': False,
'include_master': True,
'shared_folder': '/home',
'add_packages': None,
'ssh_hostbased': True,
'verboseLevel': 0
})
_v_configHolder = ConfigHolder(options)
__idMap = {}
__vmMap = {}
def __init__(
self,
name="Dummy StratusLab Cluster",
host="localhost",
cloud_type="StratusLab",
memory=None,
max_vm_mem=-1,
cpu_archs=None,
networks=None,
vm_slots=0,
cpu_cores=0,
storage=0,
contextualization='',
enabled=True,
priority=0,
keep_alive=0,
):
# Call super class' init
cluster_tools.ICluster.__init__(
self,
name=name,
host=host,
cloud_type=cloud_type,
memory=memory,
max_vm_mem=max_vm_mem,
cpu_archs=cpu_archs,
networks=networks,
vm_slots=vm_slots,
cpu_cores=cpu_cores,
storage=storage,
enabled=enabled,
priority=priority,
keep_alive=keep_alive,
)
try:
contex = open(contextualization, 'r')
strat = contex.read()
StratusLabCluster._v_configHolder.set('extraContextData', 'EC2_USER_DATA=%s' \
%base64.standard_b64encode(strat))
except IOError:
self.log.error(
"Contextualization file '%s' is not valid. \
Proceeding without contextualization...",
str(contextualization))
self.__runnerIds = {}
def vm_create(self,
vm_name,
vm_type="CernVM",
vm_user="******",
vm_networkassoc="",
vm_image="",
vm_mem=1,
vm_cores=1,
vm_storage=30,
vm_keepalive=0,
myproxy_creds_name=None,
myproxy_server=None,
myproxy_server_port=None,
job_per_core=False,
vm_loc=''):
self.log.debug(
"Running new instance with Marketplace id %s in StratusLab",
str(vm_loc))
runner = None
if vm_loc not in StratusLabCluster.__idMap:
#vm_loc: URL of VM or key? Does not seem to matter in Runner (l.506)
runner = Runner(vm_loc, StratusLabCluster._v_configHolder)
StratusLabCluster.__idMap[vm_loc] = runner
else:
runner = StratusLabCluster.__idMap[vm_loc]
try:
ids = runner.runInstance()
self.log.debug("Created instances: %s", str(ids))
#for new_id in ids:
new_id = ids[len(ids) - 1]
#if job didnt't set a keep_alive use the clouds default
if not vm_keepalive and self.keep_alive:
vm_keepalive = self.keep_alive
new_vm = cluster_tools.VM(name=vm_name,
id=str(new_id),
vmtype=vm_type,
user=vm_user,
network=vm_networkassoc,
image=vm_image,
memory=vm_mem,
cpucores=vm_cores,
storage=vm_storage,
keep_alive=vm_keepalive,
myproxy_creds_name=myproxy_creds_name,
myproxy_server=myproxy_server,
myproxy_server_port=myproxy_server_port,
job_per_core=job_per_core)
StratusLabCluster.__vmMap[str(new_id)] = vm_loc
if vm_loc not in self.__runnerIds:
self.__runnerIds[vm_loc] = [
str(new_id),
]
else:
self.__runnerIds[vm_loc].append(str(new_id))
self.vms.append(new_vm)
try:
self.resource_checkout(new_vm)
except:
self.log.exception(
"Unexpected error checking out resources when creating a VM."
" Programming error?")
return self.ERROR
#endfor
return 0
except Exception, e:
self.log.exception(
"Exception running new instance in StratusLab: %s", str(e))
return -1
def vm_create(self,
vm_name,
vm_type="CernVM",
vm_user="******",
vm_networkassoc="",
vm_image="",
vm_mem=1,
vm_cores=1,
vm_storage=30,
vm_keepalive=0,
myproxy_creds_name=None,
myproxy_server=None,
myproxy_server_port=None,
job_per_core=False,
vm_loc=''):
self.log.debug(
"Running new instance with Marketplace id %s in StratusLab",
str(vm_loc))
runner = None
if vm_loc not in StratusLabCluster.__idMap:
#vm_loc: URL of VM or key? Does not seem to matter in Runner (l.506)
runner = Runner(vm_loc, StratusLabCluster._v_configHolder)
StratusLabCluster.__idMap[vm_loc] = runner
else:
runner = StratusLabCluster.__idMap[vm_loc]
try:
ids = runner.runInstance()
self.log.debug("Created instances: %s", str(ids))
#for new_id in ids:
new_id = ids[len(ids) - 1]
#if job didnt't set a keep_alive use the clouds default
if not vm_keepalive and self.keep_alive:
vm_keepalive = self.keep_alive
new_vm = cluster_tools.VM(name=vm_name,
id=str(new_id),
vmtype=vm_type,
user=vm_user,
network=vm_networkassoc,
image=vm_image,
memory=vm_mem,
cpucores=vm_cores,
storage=vm_storage,
keep_alive=vm_keepalive,
myproxy_creds_name=myproxy_creds_name,
myproxy_server=myproxy_server,
myproxy_server_port=myproxy_server_port,
job_per_core=job_per_core)
StratusLabCluster.__vmMap[str(new_id)] = vm_loc
if vm_loc not in self.__runnerIds:
self.__runnerIds[vm_loc] = [
str(new_id),
]
else:
self.__runnerIds[vm_loc].append(str(new_id))
self.vms.append(new_vm)
try:
self.resource_checkout(new_vm)
except:
self.log.exception(
"Unexpected error checking out resources when creating a VM."
" Programming error?")
return self.ERROR
#endfor
return 0
except Exception, e:
self.log.exception(
"Exception running new instance in StratusLab: %s", str(e))
return -1
def vm_create(self,
vm_name,
vm_type="CernVM",
vm_user="******",
vm_networkassoc="",
vm_cpuarch="",
vm_image="",
vm_mem=1,
vm_cores=1,
vm_storage=30,
customization=None,
vm_keepalive=0,
job_proxy_file_path=None,
myproxy_creds_name=None,
myproxy_server=None,
myproxy_server_port=None,
job_per_core=False,
proxy_non_boot=False,
vmimage_proxy_file=None,
vmimage_proxy_file_path=None,
vm_loc=''):
log.debug("Running new instance with Marketplace id %s in StratusLab" %
str(vm_loc))
runner = None
if vm_loc not in StratusLabCluster.__idMap:
runner = Runner(
vm_loc, StratusLabCluster._v_configHolder
) #vm_loc: URL of VM or key? Does not seem to matter in Runner (l. 506)
StratusLabCluster.__idMap[vm_loc] = runner
else:
runner = StratusLabCluster.__idMap[vm_loc]
try:
ids = runner.runInstance()
log.debug("Created instances: %s" % str(ids))
#for new_id in ids:
new_id = ids[len(ids) - 1]
new_vm = cluster_tools.VM(name=vm_name,
id=str(new_id),
vmtype=vm_type,
user=vm_user,
network=vm_networkassoc,
cpuarch=vm_cpuarch,
image=vm_image,
memory=vm_mem,
cpucores=vm_cores,
storage=vm_storage,
keep_alive=vm_keepalive,
myproxy_creds_name=myproxy_creds_name,
myproxy_server=myproxy_server,
myproxy_server_port=myproxy_server_port,
job_per_core=job_per_core)
StratusLabCluster.__vmMap[str(new_id)] = vm_loc
if vm_loc not in self.__runnerIds:
self.__runnerIds[vm_loc] = [
str(new_id),
]
else:
self.__runnerIds[vm_loc].append(str(new_id))
self.vms.append(new_vm)
try:
self.resource_checkout(new_vm)
except:
log.exception(
"Unexpected error checking out resources when creating a VM. Programming error?"
)
return self.ERROR
#endfor
return 0
except Exception, e:
log.debug("Exception running new instance in StratusLab: %s" %
str(e))
#import traceback
#traceback.print_exc()
return -1
