def __init__(self, job_pool): threading.Thread.__init__(self, name=self.__class__.__name__) self.job_pool = job_pool self.quit = False self.heart_beat = time.time() # proxy expiry time poll interval, in seconds self.polling_interval = config.config_options.getint('global', 'job_proxy_refresher_interval') self.log = utilities.get_cloudscheduler_logger()
def __init__(self, cloud_resources): """ Constructor for Refreshing VM Proxies - probably not used due to Nimbus. :param cloud_resources: """ threading.Thread.__init__(self, name=self.__class__.__name__) self.cloud_resources = cloud_resources self.quit = False self.heart_beat = time.time() # proxy expiry time poll interval, in seconds self.polling_interval = config.config_options.getint('global', 'vm_proxy_refresher_interval') self.log = utilities.get_cloudscheduler_logger()
import os import sys import time import string import json import shutil import logging import nimbus_xml import subprocess import cluster_tools import cloud_init_util import cloudscheduler.config as config import cloudscheduler.utilities as utilities import datetime as dt from cloudscheduler.job_management import _attr_list_to_dict log = utilities.get_cloudscheduler_logger() try: import boto.ec2 import boto except ImportError: log.error("To use EC2-style clouds, you need to have boto " \ "installed. You can install it from your package manager, " \ "or get it from http://code.google.com/p/boto/") from httplib import BadStatusLine from subprocess import Popen from urlparse import urlparse from cStringIO import StringIO import gzip
""" from __future__ import with_statement import time import uuid import string import logging import datetime import threading import requests from cloudscheduler import config import cloudscheduler.utilities as utilities from cloudscheduler.utilities import get_cert_expiry_time log = utilities.get_cloudscheduler_logger() config_val = config.setup() class VM(object): """ A class for storing created VM information. Used to populate Cluster classes 'vms' lists. Instance Variables The global VM states are: Starting - The VM is being created in the cloud Running - The VM is running somewhere on the cloud (fully functional) Error - The VM has been corrupted or is in the process of being destroyed
class StratusLabCluster(cluster_tools.ICluster): """ Connector class for StratusLab Clouds - Has not been tested for years. """ VM_TARGETSTATE = "Running" VM_NODES = "1" VM_SHUTDOWN = 90.0 ERROR = 1 VM_STATES = { 'INIT': 'Starting', 'BOOT': 'Starting', 'PROLOG': 'Starting', 'PENDING': 'Starting', 'HOLD': 'Starting', 'RUNNING': 'Running', 'ACTIVE': 'Running', 'STOPPED': 'Running', 'SUSPENDED': 'Running', 'DONE': 'Shutdown', 'EPILOG': 'Shutdown', 'FAILED': 'Error', 'FAILURE': 'Error', 'UNKNOWN': 'Error', } log = utilities.get_cloudscheduler_logger() config = ConfigParser.ConfigParser() config_file = os.path.expanduser('~/.stratuslab/stratuslab-user.cfg') config_section = 'default' config.read(config_file) #Read the stratuslab configuration file username, password, endpoint = None, None, None try: username = config.get(config_section, 'username') except ConfigParser.NoOptionError: log.error( "Stratuslab config file %s does not contain username. VM creation will fail", config_file) except ConfigParser.NoSectionError: log.error( "No section '%s' found Stratuslab config file '%s'. \ Do the file and section exist? VM creation will fail", config_section, config_file) try: password = config.get(config_section, 'password') except ConfigParser.NoOptionError: log.error( "Stratuslab config file %s does not contain password. VM creation will fail", config_file) except ConfigParser.NoSectionError: log.error( "No section '%s' found Stratuslab config file '%s'. \ Do the file and section exist? VM creation will fail", config_section, config_file) try: endpoint = config.get(config_section, 'endpoint') except ConfigParser.NoOptionError: log.error( "Stratuslab config file %s does not contain endpoint. VM creation will fail", config_file) except ConfigParser.NoSectionError: log.error( "No section '%s' found Stratuslab config file '%s'. \ Do the file and section exist? VM creation will fail", config_section, config_file) options = Runner.defaultRunOptions() options.update({ 'username': username, 'password': password, 'endpoint': endpoint, 'marketplaceEndpoint': 'https://marketplace.stratuslab.eu', 'mpi_machine_file': True, 'cluster_admin': 'root', 'master_vmid': None, 'tolerate_failures': False, 'clean_after_failure': False, 'include_master': True, 'shared_folder': '/home', 'add_packages': None, 'ssh_hostbased': True, 'verboseLevel': 0 }) _v_configHolder = ConfigHolder(options) __idMap = {} __vmMap = {} def __init__( self, name="Dummy StratusLab Cluster", host="localhost", cloud_type="StratusLab", memory=None, max_vm_mem=-1, cpu_archs=None, networks=None, vm_slots=0, cpu_cores=0, storage=0, contextualization='', enabled=True, priority=0, keep_alive=0, ): # Call super class' init cluster_tools.ICluster.__init__( self, name=name, host=host, cloud_type=cloud_type, memory=memory, max_vm_mem=max_vm_mem, cpu_archs=cpu_archs, networks=networks, vm_slots=vm_slots, cpu_cores=cpu_cores, storage=storage, enabled=enabled, priority=priority, keep_alive=keep_alive, ) try: contex = open(contextualization, 'r') strat = contex.read() StratusLabCluster._v_configHolder.set('extraContextData', 'EC2_USER_DATA=%s' \ %base64.standard_b64encode(strat)) except IOError: self.log.error( "Contextualization file '%s' is not valid. \ Proceeding without contextualization...", str(contextualization)) self.__runnerIds = {} def vm_create(self, vm_name, vm_type="CernVM", vm_user="******", vm_networkassoc="", vm_image="", vm_mem=1, vm_cores=1, vm_storage=30, vm_keepalive=0, myproxy_creds_name=None, myproxy_server=None, myproxy_server_port=None, job_per_core=False, vm_loc=''): self.log.debug( "Running new instance with Marketplace id %s in StratusLab", str(vm_loc)) runner = None if vm_loc not in StratusLabCluster.__idMap: #vm_loc: URL of VM or key? Does not seem to matter in Runner (l.506) runner = Runner(vm_loc, StratusLabCluster._v_configHolder) StratusLabCluster.__idMap[vm_loc] = runner else: runner = StratusLabCluster.__idMap[vm_loc] try: ids = runner.runInstance() self.log.debug("Created instances: %s", str(ids)) #for new_id in ids: new_id = ids[len(ids) - 1] #if job didnt't set a keep_alive use the clouds default if not vm_keepalive and self.keep_alive: vm_keepalive = self.keep_alive new_vm = cluster_tools.VM(name=vm_name, id=str(new_id), vmtype=vm_type, user=vm_user, network=vm_networkassoc, image=vm_image, memory=vm_mem, cpucores=vm_cores, storage=vm_storage, keep_alive=vm_keepalive, myproxy_creds_name=myproxy_creds_name, myproxy_server=myproxy_server, myproxy_server_port=myproxy_server_port, job_per_core=job_per_core) StratusLabCluster.__vmMap[str(new_id)] = vm_loc if vm_loc not in self.__runnerIds: self.__runnerIds[vm_loc] = [ str(new_id), ] else: self.__runnerIds[vm_loc].append(str(new_id)) self.vms.append(new_vm) try: self.resource_checkout(new_vm) except: self.log.exception( "Unexpected error checking out resources when creating a VM." " Programming error?") return self.ERROR #endfor return 0 except Exception, e: self.log.exception( "Exception running new instance in StratusLab: %s", str(e)) return -1
def __init__(self): self.log = utilities.get_cloudscheduler_logger()