def __init__(self, cfg):
ssl = cfg.phantom.system.rabbit_ssl
self._rabbit = cfg.phantom.system.rabbit
self._rabbit_port = cfg.phantom.system.rabbit_port
self._rabbitpw = cfg.phantom.system.rabbit_pw
self._rabbituser = cfg.phantom.system.rabbit_user
self._rabbitexchange = cfg.phantom.system.rabbit_exchange
log(
logging.INFO,
"Connecting to epu messaging fabric: %s, %s, XXXXX, %d, ssl=%s" %
(self._rabbit, self._rabbituser, self._rabbit_port, str(ssl)))
self._dashi_conn = DashiCeiConnection(self._rabbit,
self._rabbituser,
self._rabbitpw,
exchange=self._rabbitexchange,
timeout=60,
port=self._rabbit_port,
ssl=ssl)
try:
self._opentsdb_host = cfg.phantom.sensor.opentsdb.host
except AttributeError:
self._opentsdb_host = DEFAULT_OPENTSDB_HOST
try:
self._opentsdb_port = cfg.phantom.sensor.opentsdb.port
except AttributeError:
self._opentsdb_port = DEFAULT_OPENTSDB_PORT
self._epum_client = EPUMClient(self._dashi_conn)
self._dtrs_client = DTRSClient(self._dashi_conn)
load_known_definitions(self._epum_client)
def __init__(self, username):
self.username = username
phantom_user = PhantomUser.objects.get(username=username)
if phantom_user is None:
msg = 'The user %s is not associated with an access key ID. Please contact your sysadmin' % (
username)
raise PhantomWebException(msg)
self.access_key = phantom_user.access_key_id
rabbit_info_objects = RabbitInfoDB.objects.all()
if not rabbit_info_objects:
raise PhantomWebException(
'The service is mis-configured. Please contact your sysadmin')
self.rabbit_info = rabbit_info_objects[0]
ssl = self.rabbit_info.rabbitssl
self._dashi_conn = DashiCeiConnection(
self.rabbit_info.rabbithost,
self.rabbit_info.rabbituser,
self.rabbit_info.rabbitpassword,
exchange=self.rabbit_info.rabbitexchange,
timeout=60,
port=self.rabbit_info.rabbitport,
ssl=ssl)
self.epum = EPUMClient(self._dashi_conn)
self.dtrs = DTRSClient(self._dashi_conn)
def add_site(self, site_name, access_key, secret_key, key_name):
cred_client = DTRSClient(self._dashi_conn)
site_credentials = {
'access_key': access_key,
'secret_key': secret_key,
'key_name': key_name
}
if site_name in self.iaasclouds:
cred_client.update_credentials(self._user_dbobject.access_key, site_name, site_credentials)
else:
cred_client.add_credentials(self._user_dbobject.access_key, site_name, site_credentials)
self._load_clouds()
def _load_clouds(self):
dtrs_client = DTRSClient(self._dashi_conn)
sites = dtrs_client.list_credentials(self._user_dbobject.access_key)
self.iaasclouds = {}
for site_name in sites:
try:
site_desc = dtrs_client.describe_site(site_name)
desc = dtrs_client.describe_credentials(self._user_dbobject.access_key, site_name)
uci = UserCloudInfo(site_name, self._user_dbobject.displayname, desc['access_key'], desc['secret_key'], desc['key_name'], site_desc['driver_class'], site_desc=site_desc)
self.iaasclouds[site_name] = uci
except Exception, ex:
g_general_log.error("Failed trying to add the site %s to the user %s | %s" % (site_name, self._user_dbobject.displayname, str(ex)))
def __init__(self, username):
self.username = username
phantom_info_objects = PhantomInfoDB.objects.all()
if not phantom_info_objects:
raise PhantomWebException('The service is mis-configured. Please contact your sysadmin')
rabbit_info_objects = RabbitInfoDB.objects.all()
if not rabbit_info_objects:
raise PhantomWebException('The service is mis-configured. Please contact your sysadmin')
self.phantom_info = phantom_info_objects[0]
self.rabbit_info = rabbit_info_objects[0]
g_general_log.debug("Using dburl %s" % (self.phantom_info.dburl))
self._authz = PhantomSQL(self.phantom_info.dburl)
self._user_dbobject = self._authz.get_user_object_by_display_name(username)
if not self._user_dbobject:
raise PhantomWebException('The user %s is not associated with cloud user database. Please contact your sysadmin' % (username))
self.access_key = self._user_dbobject.access_key
ssl = self.rabbit_info.rabbitssl
self._dashi_conn = DashiCeiConnection(self.rabbit_info.rabbithost, self.rabbit_info.rabbituser, self.rabbit_info.rabbitpassword, exchange=self.rabbit_info.rabbitexchange, timeout=60, port=self.rabbit_info.rabbitport, ssl=ssl)
self.epum = EPUMClient(self._dashi_conn)
self.dtrs = DTRSClient(self._dashi_conn)
self._load_clouds()
def main():
if len(sys.argv) != 4:
print "usage: test_add_user name access_key access_secret"
sys.exit(1)
name = sys.argv[1]
access_key = sys.argv[2]
access_secret = sys.argv[3]
ssh_key = get_user_public_key()
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.set('Boto', 'http_socket_timeout', '10')
boto.config.set('Boto', 'num_retries', '3')
dashi_con = get_dashi_client()
dtrs_client = DTRSClient(dashi_con)
add_one_user(dtrs_client, access_key, access_secret, ssh_key, name)
def add_site(self, site_name, access_key, secret_key, key_name):
cred_client = DTRSClient(self._dashi_conn)
site_credentials = {
'access_key': access_key,
'secret_key': secret_key,
'key_name': key_name
}
self.get_clouds()
if site_name in self.iaasclouds:
cred_client.update_credentials(self.access_key, site_name,
site_credentials)
else:
cred_client.add_credentials(self.access_key, site_name,
site_credentials)
self._load_clouds()
def main():
if len(sys.argv) != 4:
print "usage: add_users <user pattern> <nimbus home> <phantom conf file>"
sys.exit(1)
userpattern = sys.argv[1]
nh = sys.argv[2]
os.environ['PHANTOM_CONFIG'] = sys.argv[3]
user_pw_list = get_fg_users(userpattern, nh)
cfg = build_cfg()
authz = cfg.get_authz()
dashi_con = get_dashi_client(cfg._CFG)
dtrs_client = DTRSClient(dashi_con)
for (name, access_key, access_secret) in user_pw_list:
print "handling user %s" % (name)
(email, ssh_key) = get_user_public_key(name)
add_one_user(authz, dtrs_client, access_key, access_secret, ssh_key,
email, name)
def __init__(self, username):
self.username = username
phantom_user = PhantomUser.objects.get(username=username)
if phantom_user is None:
msg = 'The user %s is not associated with an access key ID. Please contact your sysadmin' % (username)
raise PhantomWebException(msg)
self.access_key = phantom_user.access_key_id
rabbit_info_objects = RabbitInfoDB.objects.all()
if not rabbit_info_objects:
raise PhantomWebException('The service is mis-configured. Please contact your sysadmin')
self.rabbit_info = rabbit_info_objects[0]
ssl = self.rabbit_info.rabbitssl
self._dashi_conn = DashiCeiConnection(
self.rabbit_info.rabbithost, self.rabbit_info.rabbituser, self.rabbit_info.rabbitpassword,
exchange=self.rabbit_info.rabbitexchange, timeout=60, port=self.rabbit_info.rabbitport,
ssl=ssl)
self.epum = EPUMClient(self._dashi_conn)
self.dtrs = DTRSClient(self._dashi_conn)
def __init__(self, cfg):
ssl = cfg.phantom.system.rabbit_ssl
self._rabbit = cfg.phantom.system.rabbit
self._rabbit_port = cfg.phantom.system.rabbit_port
self._rabbitpw = cfg.phantom.system.rabbit_pw
self._rabbituser = cfg.phantom.system.rabbit_user
self._rabbitexchange = cfg.phantom.system.rabbit_exchange
log(logging.INFO, "Connecting to epu messaging fabric: %s, %s, XXXXX, %d, ssl=%s" % (self._rabbit, self._rabbituser, self._rabbit_port, str(ssl)))
self._dashi_conn = DashiCeiConnection(self._rabbit, self._rabbituser, self._rabbitpw, exchange=self._rabbitexchange, timeout=60, port=self._rabbit_port, ssl=ssl)
try:
self._opentsdb_host = cfg.phantom.sensor.opentsdb.host
except AttributeError:
self._opentsdb_host = DEFAULT_OPENTSDB_HOST
try:
self._opentsdb_port = cfg.phantom.sensor.opentsdb.port
except AttributeError:
self._opentsdb_port = DEFAULT_OPENTSDB_PORT
self._epum_client = EPUMClient(self._dashi_conn)
self._dtrs_client = DTRSClient(self._dashi_conn)
load_known_definitions(self._epum_client)
def delete_site(self, site_name):
cred_client = DTRSClient(self._dashi_conn)
cred_client.remove_credentials(self.access_key, site_name)
self._load_clouds()
def get_possible_sites(self):
site_client = DTRSClient(self._dashi_conn)
l = site_client.list_sites()
return l
class EPUSystem(SystemAPI):
def __init__(self, cfg):
ssl = cfg.phantom.system.rabbit_ssl
self._rabbit = cfg.phantom.system.rabbit
self._rabbit_port = cfg.phantom.system.rabbit_port
self._rabbitpw = cfg.phantom.system.rabbit_pw
self._rabbituser = cfg.phantom.system.rabbit_user
self._rabbitexchange = cfg.phantom.system.rabbit_exchange
log(
logging.INFO,
"Connecting to epu messaging fabric: %s, %s, XXXXX, %d, ssl=%s" %
(self._rabbit, self._rabbituser, self._rabbit_port, str(ssl)))
self._dashi_conn = DashiCeiConnection(self._rabbit,
self._rabbituser,
self._rabbitpw,
exchange=self._rabbitexchange,
timeout=60,
port=self._rabbit_port,
ssl=ssl)
try:
self._opentsdb_host = cfg.phantom.sensor.opentsdb.host
except AttributeError:
self._opentsdb_host = DEFAULT_OPENTSDB_HOST
try:
self._opentsdb_port = cfg.phantom.sensor.opentsdb.port
except AttributeError:
self._opentsdb_port = DEFAULT_OPENTSDB_PORT
self._epum_client = EPUMClient(self._dashi_conn)
self._dtrs_client = DTRSClient(self._dashi_conn)
load_known_definitions(self._epum_client)
def _get_dt_details(self, name, caller):
return self._dtrs_client.describe_dt(caller, name)
def _check_dt_name_exists(self, name, caller):
name_list = self._dtrs_client.list_dts(caller=caller)
return name in name_list
@LogEntryDecorator(classname="EPUSystem")
def create_launch_config(self, user_obj, lc):
(dt_name, site_name) = _breakup_name(lc.LaunchConfigurationName)
# see if that name already exists
dt_def = None
exists = self._check_dt_name_exists(dt_name, user_obj.access_id)
if exists:
dt_def = self._get_dt_details(dt_name, user_obj.access_id)
if not dt_def:
dt_def = {}
dt_def['mappings'] = {}
if site_name in dt_def['mappings']:
raise PhantomAWSException('InvalidParameterValue',
details="Name already in use")
dt_def['mappings'][site_name] = {}
# values needed by the system
dt_def['mappings'][site_name]['iaas_allocation'] = lc.InstanceType
dt_def['mappings'][site_name]['iaas_image'] = lc.ImageId
dt_def['mappings'][site_name][
'key_name'] = phantom_get_default_key_name()
# user defined values
dt_def['mappings'][site_name]['CreatedTime'] = make_time(
lc.CreatedTime.date_time)
#dt_def['mappings'][site_name]['BlockDeviceMappings'] = lc.BlockDeviceMappings
dt_def['mappings'][site_name][
'InstanceMonitoring'] = lc.InstanceMonitoring.Enabled
dt_def['mappings'][site_name]['KernelId'] = lc.KernelId
dt_def['mappings'][site_name]['RamdiskId'] = lc.RamdiskId
dt_def['mappings'][site_name]['RequestedKeyName'] = lc.KeyName
dt_def['mappings'][site_name][
'LaunchConfigurationARN'] = lc.LaunchConfigurationARN
if lc.UserData:
try:
lc.UserData = base64.b64decode(lc.UserData)
except:
raise PhantomAWSException(
'InvalidParameterValue',
details="UserData should be base64-encoded")
dt_def['contextualization'] = {}
dt_def['contextualization']['method'] = 'userdata'
dt_def['contextualization']['userdata'] = lc.UserData
if exists:
self._dtrs_client.update_dt(user_obj.access_id, dt_name, dt_def)
else:
self._dtrs_client.add_dt(user_obj.access_id, dt_name, dt_def)
@LogEntryDecorator(classname="EPUSystem")
def delete_launch_config(self, user_obj, name):
(dt_name, site_name) = _breakup_name(name)
dt_def = self._get_dt_details(dt_name, user_obj.access_id)
if not dt_def:
raise PhantomAWSException('InvalidParameterValue',
details="Name %s not found" % (name))
if site_name not in dt_def['mappings']:
raise PhantomAWSException('InvalidParameterValue',
details="Name %s not found" % (name))
del dt_def['mappings'][site_name]
if len(dt_def['mappings']) == 0:
self._dtrs_client.remove_dt(user_obj.access_id, dt_name)
else:
self._dtrs_client.update_dt(user_obj.access_id, dt_name, dt_def)
@LogEntryDecorator(classname="EPUSystem")
def get_launch_configs(self,
user_obj,
names=None,
max=-1,
startToken=None):
next_token = None
dts = self._dtrs_client.list_dts(user_obj.access_id)
dts.sort()
# now that we have the final list, look up each description
lc_list_type = AWSListType('LaunchConfigurations')
for lc_name in dts:
if lc_list_type.get_length() >= max and max > -1:
break
dt_descr = self._get_dt_details(lc_name, user_obj.access_id)
for site in sorted(dt_descr['mappings'].keys()):
mapped_def = dt_descr['mappings'][site]
out_name = '%s@%s' % (lc_name, site)
if lc_list_type.get_length() >= max and max > -1:
break
if out_name == startToken:
startToken = None
if startToken is None and (names is None or out_name in names):
ot_lc = LaunchConfigurationType('LaunchConfiguration')
ot_lc.BlockDeviceMappings = AWSListType(
'BlockDeviceMappings')
if 'CreatedTime' not in mapped_def.keys():
# This is an LC that was created with the new Phantom API, ignore it
continue
tm = _get_time(mapped_def['CreatedTime'])
ot_lc.CreatedTime = DateTimeType('CreatedTime', tm)
ot_lc.ImageId = mapped_def['iaas_image']
ot_lc.InstanceMonitoring = InstanceMonitoringType(
'InstanceMonitoring')
ot_lc.InstanceMonitoring.Enabled = False
ot_lc.InstanceType = mapped_def['iaas_allocation']
ot_lc.KernelId = None
ot_lc.KeyName = phantom_get_default_key_name()
ot_lc.LaunchConfigurationARN = mapped_def[
'LaunchConfigurationARN']
ot_lc.LaunchConfigurationName = out_name
ot_lc.RamdiskId = None
ot_lc.SecurityGroups = AWSListType('SecurityGroups')
contextualization = dt_descr.get('contextualization')
if contextualization is not None and contextualization.get(
'method') == 'userdata':
# UserData should be base64-encoded to be properly decoded by boto
ot_lc.UserData = base64.b64encode(
contextualization.get('userdata'))
else:
ot_lc.UserData = None
lc_list_type.add_item(ot_lc)
# XXX need to set next_token
return (lc_list_type, next_token)
@LogEntryDecorator(classname="EPUSystem")
def create_autoscale_group(self, user_obj, asg):
log(
logging.DEBUG, "entering create_autoscale_group with %s" %
(asg.LaunchConfigurationName))
(definition_name, domain_opts) = tags_to_definition(asg.Tags.type_list)
domain_opts['minimum_vms'] = asg.DesiredCapacity
dt_name = asg.LaunchConfigurationName
site_name = ""
if dt_name.find('@') > 0:
(dt_name, site_name) = _breakup_name(asg.LaunchConfigurationName)
if definition_name == 'sensor_engine':
domain_opts['deployable_type'] = dt_name
domain_opts['dtname'] = dt_name
domain_opts['iaas_site'] = site_name
domain_opts['iaas_allocation'] = "m1.small"
domain_opts['minimum_vms'] = asg.MinSize
domain_opts['maximum_vms'] = asg.MaxSize
domain_opts['opentsdb_host'] = self._opentsdb_host
domain_opts['opentsdb_port'] = self._opentsdb_port
else:
domain_opts['dtname'] = dt_name
domain_opts['minimum_vms'] = asg.MinSize
domain_opts['maximum_vms'] = asg.MaxSize
domain_opts['opentsdb_host'] = self._opentsdb_host
domain_opts['opentsdb_port'] = self._opentsdb_port
#domain_opts['force_site'] = site_name
domain_opts['CreatedTime'] = make_time(asg.CreatedTime.date_time)
domain_opts['AutoScalingGroupARN'] = asg.AutoScalingGroupARN
domain_opts['VPCZoneIdentifier'] = asg.VPCZoneIdentifier
domain_opts['HealthCheckType'] = asg.HealthCheckType
domain_opts['PlacementGroup'] = asg.PlacementGroup
conf = {'engine_conf': domain_opts}
log(logging.INFO, "Creating autoscale group with %s" % (conf))
try:
self._epum_client.add_domain(asg.AutoScalingGroupName,
definition_name,
conf,
caller=user_obj.access_id)
except DashiError, de:
if de.exc_type == u'WriteConflictError':
raise PhantomAWSException(
'InvalidParameterValue',
details="auto scale name already exists")
log(logging.ERROR, "An error creating ASG: %s" % (str(de)))
raise
def delete_site(self, site_name):
cred_client = DTRSClient(self._dashi_conn)
cred_client.remove_credentials(self.access_key, site_name)
self._load_clouds()
def get_possible_sites(self, details=False):
site_client = DTRSClient(self._dashi_conn)
site_names = site_client.list_sites(self.access_key)
all_sites = {}
for site in site_names:
all_sites[site] = {'id': site, 'instance_types': INSTANCE_TYPES}
if details is True:
pool = Pool()
instance_types_results = {}
for site in site_names:
instance_types_results[site] = pool.apply_async(site_client.describe_site, [self.access_key, site])
pool.close()
for site in site_names:
try:
site_description = instance_types_results[site].get(IAAS_TIMEOUT)
instance_types = site_description.get("instance_types")
if instance_types is not None:
all_sites[site]['instance_types'] = instance_types
cloud_type = site_description.get("type")
if cloud_type is not None:
all_sites[site]['type'] = cloud_type
image_generation = site_description.get("image_generation")
if image_generation:
all_sites[site]['image_generation'] = True
else:
all_sites[site]['image_generation'] = False
except TimeoutError:
log.exception("Timed out getting list of instance types for %s" % site)
except Exception:
log.exception("Unexpected error getting list of instance types for %s" % site)
pool = Pool()
public_results = {}
user_results = {}
clouds = self.get_clouds()
for cloud_name, cloud in clouds.iteritems():
result = pool.apply_async(cloud.get_user_images)
user_results[cloud_name] = result
if cloud.site_desc["type"] != "ec2":
result = pool.apply_async(cloud.get_public_images)
public_results[cloud_name] = result
pool.close()
for cloud_name, result in user_results.iteritems():
try:
all_sites[cloud_name]['user_images'] = result.get(IAAS_TIMEOUT)
except TimeoutError:
log.exception("Timed out getting images from %s" % cloud_name)
all_sites[cloud_name]['user_images'] = []
except Exception:
log.exception("Unexpected error getting images from %s" % cloud_name)
all_sites[cloud_name]['user_images'] = []
for cloud_name, result in public_results.iteritems():
try:
all_sites[cloud_name]['public_images'] = result.get(IAAS_TIMEOUT)
except TimeoutError:
log.exception("Timed out getting images from %s" % cloud_name)
all_sites[cloud_name]['public_images'] = []
except Exception:
log.exception("Unexpected error getting images from %s" % cloud_name)
all_sites[cloud_name]['public_images'] = []
return all_sites
class UserObjectMySQL(UserObject):
def __init__(self, username):
self.username = username
phantom_user = PhantomUser.objects.get(username=username)
if phantom_user is None:
msg = 'The user %s is not associated with an access key ID. Please contact your sysadmin' % (username)
raise PhantomWebException(msg)
self.access_key = phantom_user.access_key_id
rabbit_info_objects = RabbitInfoDB.objects.all()
if not rabbit_info_objects:
raise PhantomWebException('The service is mis-configured. Please contact your sysadmin')
self.rabbit_info = rabbit_info_objects[0]
ssl = self.rabbit_info.rabbitssl
self._dashi_conn = DashiCeiConnection(
self.rabbit_info.rabbithost, self.rabbit_info.rabbituser, self.rabbit_info.rabbitpassword,
exchange=self.rabbit_info.rabbitexchange, timeout=60, port=self.rabbit_info.rabbitport,
ssl=ssl)
self.epum = EPUMClient(self._dashi_conn)
self.dtrs = DTRSClient(self._dashi_conn)
def describe_domain(self, username, domain):
describe = self.epum.describe_domain(domain, caller=self.access_key)
return describe
def remove_domain(self, username, domain):
removed = self.epum.remove_domain(domain, caller=self.access_key)
return removed
def _api_parameters_to_general_opts(self, parameters):
general_opts = {}
if 'chef_credential' in parameters:
general_opts['chef_credential'] = parameters['chef_credential']
return general_opts
def _api_parameters_to_domain_opts(self, parameters):
domain_opts = {}
de_name = parameters.get('de_name')
domain_opts['phantom_de_name'] = de_name
domain_opts['clouds'] = parameters.get('clouds', [])
if de_name == 'sensor':
try:
domain_opts['dtname'] = parameters['lc_name']
domain_opts['cooldown_period'] = parameters['sensor_cooldown']
domain_opts['maximum_vms'] = parameters['sensor_maximum_vms']
domain_opts['minimum_vms'] = parameters['sensor_minimum_vms']
domain_opts['metric'] = parameters['sensor_metric']
domain_opts['monitor_domain_sensors'] = parameters.get('monitor_domain_sensors', '').split(',')
domain_opts['monitor_sensors'] = parameters.get('monitor_sensors', '').split(',')
domain_opts['scale_down_n_vms'] = parameters['sensor_scale_down_vms']
domain_opts['scale_down_threshold'] = parameters['sensor_scale_down_threshold']
domain_opts['scale_up_n_vms'] = parameters['sensor_scale_up_vms']
domain_opts['scale_up_threshold'] = parameters['sensor_scale_up_threshold']
domain_opts['sample_function'] = 'Average' # TODO: make configurable
domain_opts['sensor_type'] = 'opentsdb' # TODO: make configurable
domain_opts['opentsdb_port'] = int(settings.OPENTSDB_PORT)
domain_opts['opentsdb_host'] = settings.OPENTSDB_HOST
except KeyError as k:
raise PhantomWebException("Mandatory parameter '%s' is missing" % k.args[0])
elif de_name == 'multicloud':
try:
domain_opts['dtname'] = parameters['lc_name']
domain_opts['maximum_vms'] = parameters['vm_count']
domain_opts['minimum_vms'] = parameters['vm_count']
domain_opts['monitor_domain_sensors'] = parameters.get('monitor_domain_sensors', '').split(',')
domain_opts['monitor_sensors'] = parameters.get('monitor_sensors', '').split(',')
domain_opts['sample_function'] = 'Average' # TODO: make configurable
domain_opts['sensor_type'] = 'opentsdb' # TODO: make configurable
domain_opts['opentsdb_port'] = int(settings.OPENTSDB_PORT)
domain_opts['opentsdb_host'] = settings.OPENTSDB_HOST
except KeyError as k:
raise PhantomWebException("Mandatory parameter '%s' is missing" % k.args[0])
else:
raise PhantomWebException("de_name '%s' is not supported" % de_name)
return domain_opts
def add_domain(self, username, name, parameters):
domain_opts = self._api_parameters_to_domain_opts(parameters)
general_opts = self._api_parameters_to_general_opts(parameters)
id = str(uuid4())
parameters['id'] = id
domain_opts['name'] = id
domain_opts['phantom_name'] = name
conf = {'engine_conf': domain_opts, 'general': general_opts}
try:
self.epum.add_domain(id, PHANTOM_DOMAIN_DEFINITION, conf, caller=self.access_key)
return parameters
except Exception:
log.exception("Problem creating domain: %s" % name)
raise
def reconfigure_domain(self, username, id, parameters):
name = parameters.get('name')
domain_opts = self._api_parameters_to_domain_opts(parameters)
general_opts = self._api_parameters_to_general_opts(parameters)
parameters['id'] = id
domain_opts['name'] = id
domain_opts['phantom_name'] = name
conf = {'engine_conf': domain_opts, 'general': general_opts}
try:
self.epum.reconfigure_domain(id, conf, caller=self.access_key)
return parameters
except Exception:
log.exception("Problem modifying domain: %s" % name)
raise
def get_all_domains(self, username):
domain_names = self.epum.list_domains(caller=self.access_key)
return domain_names
def _sanitize_sensor_data(self, sensor_data):
cleaned_sensor_data = {}
for key, item in sensor_data.iteritems():
key = key.lower()
cleaned_item = {}
for k, v in item.iteritems():
k = k.lower()
cleaned_item[k] = v
cleaned_sensor_data[key] = cleaned_item
return cleaned_sensor_data
def get_domain(self, username, id):
try:
domain_description = self.epum.describe_domain(id, caller=self.access_key)
except DashiError:
return None
engine_conf = domain_description.get('config', {}).get('engine_conf', {})
general_conf = domain_description.get('config', {}).get('general', {})
ent = {}
ent['id'] = domain_description['name']
ent['de_name'] = engine_conf.get('phantom_de_name')
ent['name'] = engine_conf.get('phantom_name')
ent['sensor_data'] = self._sanitize_sensor_data(domain_description.get('sensor_data', {}))
ent['monitor_sensors'] = ",".join(engine_conf.get('monitor_sensors', []))
ent['monitor_domain_sensors'] = ",".join(engine_conf.get('monitor_domain_sensors', []))
if 'chef_credential' in general_conf:
ent['chef_credential'] = general_conf['chef_credential']
if ent['de_name'] == 'multicloud':
ent['vm_count'] = engine_conf.get('minimum_vms')
ent['lc_name'] = engine_conf.get('dtname')
elif ent['de_name'] == 'sensor':
ent['lc_name'] = engine_conf.get('dtname')
ent['sensor_minimum_vms'] = engine_conf.get('minimum_vms')
ent['sensor_maximum_vms'] = engine_conf.get('maximum_vms')
ent['sensor_metric'] = engine_conf.get('metric')
ent['sensor_scale_down_threshold'] = engine_conf.get('scale_down_threshold')
ent['sensor_scale_down_vms'] = engine_conf.get('scale_down_n_vms')
ent['sensor_scale_up_threshold'] = engine_conf.get('scale_up_threshold')
ent['sensor_scale_up_vms'] = engine_conf.get('scale_up_n_vms')
ent['sensor_cooldown'] = engine_conf.get('cooldown_period')
return ent
def get_domain_instances(self, username, id):
try:
domain_description = self.epum.describe_domain(id, caller=self.access_key)
except DashiError:
return None
instances = domain_description.get('instances', [])
parsed_instances = []
for i in instances:
instance = {
'id': i.get('instance_id', ''),
'iaas_instance_id': i.get('iaas_id', ''),
'lifecycle_state': i.get('state', ''),
'hostname': i.get('hostname', ''),
'public_ip': i.get('public_ip', ''),
'private_ip': i.get('private_ip', ''),
'cloud': i.get('site', ''),
'image_id': i.get('iaas_image', ''),
'instance_type': i.get('iaas_allocation', ''),
'sensor_data': self._sanitize_sensor_data(i.get('sensor_data', {})),
'keyname': i.get('iaas_sshkeyname', ''),
}
parsed_instances.append(instance)
return parsed_instances
def get_all_groups(self):
domain_names = self.epum.list_domains(caller=self.access_key)
domains = []
for domain in domain_names:
domain_description = self.epum.describe_domain(domain, caller=self.access_key)
domains.append(domain_description)
return domains
def create_dt(self, dt_name, cloud_params, context_params, appliance=None):
dt = self.get_dt(dt_name)
if dt is None:
dt = {}
dt['mappings'] = {}
create = True
else:
create = False
cloud_credentials = self.get_clouds()
for cloud_name, parameters in cloud_params.iteritems():
mapping = dt['mappings'].get(cloud_name)
if mapping is None:
mapping = dt['mappings'][cloud_name] = {}
credentials = cloud_credentials.get(cloud_name, None)
# Required by EPUM
mapping['iaas_allocation'] = parameters.get('instance_type')
mapping['iaas_image'] = parameters.get('image_id')
if credentials is not None:
mapping['key_name'] = credentials.keyname
else:
# TODO: raise error?
mapping['key_name'] = ''
# Phantom stuff
mapping['common'] = parameters.get('common')
mapping['rank'] = parameters.get('rank')
mapping['max_vms'] = parameters.get('max_vms')
# Contextualization
if context_params.get('contextualization_method') == 'user_data' or context_params.get('user_data'):
contextualization = dt.get('contextualization')
if contextualization is None:
contextualization = dt['contextualization'] = {}
contextualization['method'] = 'userdata'
contextualization['userdata'] = context_params['user_data']
elif context_params.get('contextualization_method') == 'chef':
contextualization = dt.get('contextualization')
if contextualization is None:
contextualization = dt['contextualization'] = {}
if contextualization.get('userdata') is not None:
del contextualization['userdata']
contextualization['method'] = 'chef'
try:
contextualization['run_list'] = json.loads(context_params.get('chef_runlist', '[]'))
except Exception:
log.exception("Problem parsing LC content")
raise PhantomWebException("Problem parsing runlist when creating LC: %s" % context_params.get('chef_runlist'))
try:
contextualization['attributes'] = json.loads(context_params.get('chef_attributes', '{}'))
except Exception:
log.exception("Problem parsing LC content")
raise PhantomWebException("Problem parsing chef attributes when creating LC: %s" % context_params.get('chef_attributes'))
elif context_params.get('contextualization_method') == 'none':
contextualization = dt['contextualization'] = {}
contextualization['method'] = None
if appliance is not None:
dt['appliance'] = appliance
else:
if 'appliance' in dt:
del dt['appliance']
if create:
return self.dtrs.add_dt(self.access_key, dt_name, dt)
else:
return self.dtrs.update_dt(self.access_key, dt_name, dt)
def get_dt(self, dt_name):
return self.dtrs.describe_dt(self.access_key, dt_name)
def remove_dt(self, dt_name):
return self.dtrs.remove_dt(self.access_key, dt_name)
def get_all_lcs(self):
dt_names = self.dtrs.list_dts(self.access_key)
dts = []
for dt_name in dt_names:
dt = self.dtrs.describe_dt(self.access_key, dt_name)
dts.append(dt)
return dts
def _load_clouds(self):
sites = self.dtrs.list_credentials(self.access_key)
self.iaasclouds = {}
for site_name in sites:
try:
site_desc = self.dtrs.describe_site(self.access_key, site_name)
desc = self.dtrs.describe_credentials(self.access_key, site_name)
uci = UserCloudInfo(site_name, self.username, desc['access_key'],
desc['secret_key'], desc['key_name'], site_desc)
self.iaasclouds[site_name] = uci
except Exception, ex:
log.error("Failed trying to add the site %s to the user %s | %s" % (site_name, self.username, str(ex)))
class EPUSystem(SystemAPI):
def __init__(self, cfg):
ssl = cfg.phantom.system.rabbit_ssl
self._rabbit = cfg.phantom.system.rabbit
self._rabbit_port = cfg.phantom.system.rabbit_port
self._rabbitpw = cfg.phantom.system.rabbit_pw
self._rabbituser = cfg.phantom.system.rabbit_user
self._rabbitexchange = cfg.phantom.system.rabbit_exchange
log(logging.INFO, "Connecting to epu messaging fabric: %s, %s, XXXXX, %d, ssl=%s" % (self._rabbit, self._rabbituser, self._rabbit_port, str(ssl)))
self._dashi_conn = DashiCeiConnection(self._rabbit, self._rabbituser, self._rabbitpw, exchange=self._rabbitexchange, timeout=60, port=self._rabbit_port, ssl=ssl)
try:
self._opentsdb_host = cfg.phantom.sensor.opentsdb.host
except AttributeError:
self._opentsdb_host = DEFAULT_OPENTSDB_HOST
try:
self._opentsdb_port = cfg.phantom.sensor.opentsdb.port
except AttributeError:
self._opentsdb_port = DEFAULT_OPENTSDB_PORT
self._epum_client = EPUMClient(self._dashi_conn)
self._dtrs_client = DTRSClient(self._dashi_conn)
load_known_definitions(self._epum_client)
def _get_dt_details(self, name, caller):
return self._dtrs_client.describe_dt(caller, name)
def _check_dt_name_exists(self, name, caller):
name_list = self._dtrs_client.list_dts(caller=caller)
return name in name_list
@LogEntryDecorator(classname="EPUSystem")
def create_launch_config(self, user_obj, lc):
(dt_name, site_name) = _breakup_name(lc.LaunchConfigurationName)
# see if that name already exists
dt_def = None
exists = self._check_dt_name_exists(dt_name, user_obj.access_id)
if exists:
dt_def = self._get_dt_details(dt_name, user_obj.access_id)
if not dt_def:
dt_def = {}
dt_def['mappings'] = {}
if site_name in dt_def['mappings']:
raise PhantomAWSException('InvalidParameterValue',details="Name already in use")
dt_def['mappings'][site_name] = {}
# values needed by the system
dt_def['mappings'][site_name]['iaas_allocation'] = lc.InstanceType
dt_def['mappings'][site_name]['iaas_image'] = lc.ImageId
dt_def['mappings'][site_name]['key_name'] = phantom_get_default_key_name()
# user defined values
dt_def['mappings'][site_name]['CreatedTime'] = make_time(lc.CreatedTime.date_time)
#dt_def['mappings'][site_name]['BlockDeviceMappings'] = lc.BlockDeviceMappings
dt_def['mappings'][site_name]['InstanceMonitoring'] = lc.InstanceMonitoring.Enabled
dt_def['mappings'][site_name]['KernelId'] = lc.KernelId
dt_def['mappings'][site_name]['RamdiskId'] = lc.RamdiskId
dt_def['mappings'][site_name]['RequestedKeyName'] = lc.KeyName
dt_def['mappings'][site_name]['LaunchConfigurationARN'] = lc.LaunchConfigurationARN
if lc.UserData:
try:
lc.UserData = base64.b64decode(lc.UserData)
except:
raise PhantomAWSException('InvalidParameterValue', details="UserData should be base64-encoded")
dt_def['contextualization'] = {}
dt_def['contextualization']['method'] = 'userdata'
dt_def['contextualization']['userdata'] = lc.UserData
if exists:
self._dtrs_client.update_dt(user_obj.access_id, dt_name, dt_def)
else:
self._dtrs_client.add_dt(user_obj.access_id, dt_name, dt_def)
@LogEntryDecorator(classname="EPUSystem")
def delete_launch_config(self, user_obj, name):
(dt_name, site_name) = _breakup_name(name)
dt_def = self._get_dt_details(dt_name, user_obj.access_id)
if not dt_def:
raise PhantomAWSException('InvalidParameterValue', details="Name %s not found" % (name))
if site_name not in dt_def['mappings']:
raise PhantomAWSException('InvalidParameterValue', details="Name %s not found" % (name))
del dt_def['mappings'][site_name]
if len(dt_def['mappings']) == 0:
self._dtrs_client.remove_dt(user_obj.access_id, dt_name)
else:
self._dtrs_client.update_dt(user_obj.access_id, dt_name, dt_def)
@LogEntryDecorator(classname="EPUSystem")
def get_launch_configs(self, user_obj, names=None, max=-1, startToken=None):
next_token = None
dts = self._dtrs_client.list_dts(user_obj.access_id)
dts.sort()
# now that we have the final list, look up each description
lc_list_type = AWSListType('LaunchConfigurations')
for lc_name in dts:
if lc_list_type.get_length() >= max and max > -1:
break
dt_descr = self._get_dt_details(lc_name, user_obj.access_id)
for site in sorted(dt_descr['mappings'].keys()):
mapped_def = dt_descr['mappings'][site]
out_name = '%s@%s' % (lc_name, site)
if lc_list_type.get_length() >= max and max > -1:
break
if out_name == startToken:
startToken = None
if startToken is None and (names is None or out_name in names):
ot_lc = LaunchConfigurationType('LaunchConfiguration')
ot_lc.BlockDeviceMappings = AWSListType('BlockDeviceMappings')
if 'CreatedTime' not in mapped_def.keys():
# This is an LC that was created with the new Phantom API, ignore it
continue
tm = _get_time(mapped_def['CreatedTime'])
ot_lc.CreatedTime = DateTimeType('CreatedTime', tm)
ot_lc.ImageId = mapped_def['iaas_image']
ot_lc.InstanceMonitoring = InstanceMonitoringType('InstanceMonitoring')
ot_lc.InstanceMonitoring.Enabled = False
ot_lc.InstanceType = mapped_def['iaas_allocation']
ot_lc.KernelId = None
ot_lc.KeyName = phantom_get_default_key_name()
ot_lc.LaunchConfigurationARN = mapped_def['LaunchConfigurationARN']
ot_lc.LaunchConfigurationName = out_name
ot_lc.RamdiskId = None
ot_lc.SecurityGroups = AWSListType('SecurityGroups')
contextualization = dt_descr.get('contextualization')
if contextualization is not None and contextualization.get('method') == 'userdata':
# UserData should be base64-encoded to be properly decoded by boto
ot_lc.UserData = base64.b64encode(contextualization.get('userdata'))
else:
ot_lc.UserData = None
lc_list_type.add_item(ot_lc)
# XXX need to set next_token
return (lc_list_type, next_token)
@LogEntryDecorator(classname="EPUSystem")
def create_autoscale_group(self, user_obj, asg):
log(logging.DEBUG, "entering create_autoscale_group with %s" % (asg.LaunchConfigurationName))
(definition_name, domain_opts) = tags_to_definition(asg.Tags.type_list)
domain_opts['minimum_vms'] = asg.DesiredCapacity
dt_name = asg.LaunchConfigurationName
site_name = ""
if dt_name.find('@') > 0:
(dt_name, site_name) = _breakup_name(asg.LaunchConfigurationName)
if definition_name == 'sensor_engine':
domain_opts['deployable_type'] = dt_name
domain_opts['dtname'] = dt_name
domain_opts['iaas_site'] = site_name
domain_opts['iaas_allocation'] = "m1.small"
domain_opts['minimum_vms'] = asg.MinSize
domain_opts['maximum_vms'] = asg.MaxSize
domain_opts['opentsdb_host'] = self._opentsdb_host
domain_opts['opentsdb_port'] = self._opentsdb_port
else:
domain_opts['dtname'] = dt_name
domain_opts['minimum_vms'] = asg.MinSize
domain_opts['maximum_vms'] = asg.MaxSize
domain_opts['opentsdb_host'] = self._opentsdb_host
domain_opts['opentsdb_port'] = self._opentsdb_port
#domain_opts['force_site'] = site_name
domain_opts['CreatedTime'] = make_time(asg.CreatedTime.date_time)
domain_opts['AutoScalingGroupARN'] = asg.AutoScalingGroupARN
domain_opts['VPCZoneIdentifier'] = asg.VPCZoneIdentifier
domain_opts['HealthCheckType'] = asg.HealthCheckType
domain_opts['PlacementGroup'] = asg.PlacementGroup
conf = {'engine_conf': domain_opts}
log(logging.INFO, "Creating autoscale group with %s" % (conf))
try:
self._epum_client.add_domain(asg.AutoScalingGroupName, definition_name, conf, caller=user_obj.access_id)
except DashiError, de:
if de.exc_type == u'WriteConflictError':
raise PhantomAWSException('InvalidParameterValue', details="auto scale name already exists")
log(logging.ERROR, "An error creating ASG: %s" % (str(de)))
raise
class UserObjectMySQL(UserObject):
def __init__(self, username):
self.username = username
phantom_info_objects = PhantomInfoDB.objects.all()
if not phantom_info_objects:
raise PhantomWebException('The service is mis-configured. Please contact your sysadmin')
rabbit_info_objects = RabbitInfoDB.objects.all()
if not rabbit_info_objects:
raise PhantomWebException('The service is mis-configured. Please contact your sysadmin')
self.phantom_info = phantom_info_objects[0]
self.rabbit_info = rabbit_info_objects[0]
g_general_log.debug("Using dburl %s" % (self.phantom_info.dburl))
self._authz = PhantomSQL(self.phantom_info.dburl)
self._user_dbobject = self._authz.get_user_object_by_display_name(username)
if not self._user_dbobject:
raise PhantomWebException('The user %s is not associated with cloud user database. Please contact your sysadmin' % (username))
self.access_key = self._user_dbobject.access_key
ssl = self.rabbit_info.rabbitssl
self._dashi_conn = DashiCeiConnection(self.rabbit_info.rabbithost, self.rabbit_info.rabbituser, self.rabbit_info.rabbitpassword, exchange=self.rabbit_info.rabbitexchange, timeout=60, port=self.rabbit_info.rabbitport, ssl=ssl)
self.epum = EPUMClient(self._dashi_conn)
self.dtrs = DTRSClient(self._dashi_conn)
self._load_clouds()
def close(self):
self._authz.close()
def has_phantom_data(self):
return True
def describe_domain(self, username, domain):
# TODO: this should eventually be part of the REST API
describe = self.epum.describe_domain(domain, caller=username)
g_general_log.info("describe_domain: %s" % describe)
return describe
def get_all_groups(self):
domain_names = self.epum.list_domains(caller=self.access_key)
domains = []
for domain in domain_names:
domain_description = self.epum.describe_domain(domain, caller=self.access_key)
domains.append(domain_description)
return domains
def get_all_lcs(self):
dt_names = self.dtrs.list_dts(self.access_key)
dts = []
for dt_name in dt_names:
dt = self.dtrs.describe_dt(self.access_key, dt_name)
dts.append(dt)
return dts
def load_clouds(self):
self._load_clouds()
def _load_clouds(self):
dtrs_client = DTRSClient(self._dashi_conn)
sites = dtrs_client.list_credentials(self._user_dbobject.access_key)
self.iaasclouds = {}
for site_name in sites:
try:
site_desc = dtrs_client.describe_site(site_name)
desc = dtrs_client.describe_credentials(self._user_dbobject.access_key, site_name)
uci = UserCloudInfo(site_name, self._user_dbobject.displayname, desc['access_key'], desc['secret_key'], desc['key_name'], site_desc['driver_class'], site_desc=site_desc)
self.iaasclouds[site_name] = uci
except Exception, ex:
g_general_log.error("Failed trying to add the site %s to the user %s | %s" % (site_name, self._user_dbobject.displayname, str(ex)))
class UserObjectMySQL(UserObject):
def __init__(self, username):
self.username = username
phantom_user = PhantomUser.objects.get(username=username)
if phantom_user is None:
msg = 'The user %s is not associated with an access key ID. Please contact your sysadmin' % (
username)
raise PhantomWebException(msg)
self.access_key = phantom_user.access_key_id
rabbit_info_objects = RabbitInfoDB.objects.all()
if not rabbit_info_objects:
raise PhantomWebException(
'The service is mis-configured. Please contact your sysadmin')
self.rabbit_info = rabbit_info_objects[0]
ssl = self.rabbit_info.rabbitssl
self._dashi_conn = DashiCeiConnection(
self.rabbit_info.rabbithost,
self.rabbit_info.rabbituser,
self.rabbit_info.rabbitpassword,
exchange=self.rabbit_info.rabbitexchange,
timeout=60,
port=self.rabbit_info.rabbitport,
ssl=ssl)
self.epum = EPUMClient(self._dashi_conn)
self.dtrs = DTRSClient(self._dashi_conn)
def describe_domain(self, username, domain):
describe = self.epum.describe_domain(domain, caller=self.access_key)
return describe
def remove_domain(self, username, domain):
removed = self.epum.remove_domain(domain, caller=self.access_key)
return removed
def _api_parameters_to_general_opts(self, parameters):
general_opts = {}
if 'chef_credential' in parameters:
general_opts['chef_credential'] = parameters['chef_credential']
return general_opts
def _api_parameters_to_domain_opts(self, parameters):
domain_opts = {}
de_name = parameters.get('de_name')
domain_opts['phantom_de_name'] = de_name
domain_opts['clouds'] = parameters.get('clouds', [])
if de_name == 'sensor':
try:
domain_opts['dtname'] = parameters['lc_name']
domain_opts['cooldown_period'] = parameters['sensor_cooldown']
domain_opts['maximum_vms'] = parameters['sensor_maximum_vms']
domain_opts['minimum_vms'] = parameters['sensor_minimum_vms']
domain_opts['metric'] = parameters['sensor_metric']
domain_opts['monitor_domain_sensors'] = parameters.get(
'monitor_domain_sensors', '').split(',')
domain_opts['monitor_sensors'] = parameters.get(
'monitor_sensors', '').split(',')
domain_opts['scale_down_n_vms'] = parameters[
'sensor_scale_down_vms']
domain_opts['scale_down_threshold'] = parameters[
'sensor_scale_down_threshold']
domain_opts['scale_up_n_vms'] = parameters[
'sensor_scale_up_vms']
domain_opts['scale_up_threshold'] = parameters[
'sensor_scale_up_threshold']
domain_opts[
'sample_function'] = 'Average' # TODO: make configurable
domain_opts[
'sensor_type'] = 'opentsdb' # TODO: make configurable
domain_opts['opentsdb_port'] = int(settings.OPENTSDB_PORT)
domain_opts['opentsdb_host'] = settings.OPENTSDB_HOST
except KeyError as k:
raise PhantomWebException(
"Mandatory parameter '%s' is missing" % k.args[0])
elif de_name == 'multicloud':
try:
domain_opts['dtname'] = parameters['lc_name']
domain_opts['maximum_vms'] = parameters['vm_count']
domain_opts['minimum_vms'] = parameters['vm_count']
domain_opts['monitor_domain_sensors'] = parameters.get(
'monitor_domain_sensors', '').split(',')
domain_opts['monitor_sensors'] = parameters.get(
'monitor_sensors', '').split(',')
domain_opts[
'sample_function'] = 'Average' # TODO: make configurable
domain_opts[
'sensor_type'] = 'opentsdb' # TODO: make configurable
domain_opts['opentsdb_port'] = int(settings.OPENTSDB_PORT)
domain_opts['opentsdb_host'] = settings.OPENTSDB_HOST
except KeyError as k:
raise PhantomWebException(
"Mandatory parameter '%s' is missing" % k.args[0])
else:
raise PhantomWebException("de_name '%s' is not supported" %
de_name)
return domain_opts
def add_domain(self, username, name, parameters):
domain_opts = self._api_parameters_to_domain_opts(parameters)
general_opts = self._api_parameters_to_general_opts(parameters)
id = str(uuid4())
parameters['id'] = id
domain_opts['name'] = id
domain_opts['phantom_name'] = name
conf = {'engine_conf': domain_opts, 'general': general_opts}
try:
self.epum.add_domain(id,
PHANTOM_DOMAIN_DEFINITION,
conf,
caller=self.access_key)
return parameters
except Exception:
log.exception("Problem creating domain: %s" % name)
raise
def reconfigure_domain(self, username, id, parameters):
name = parameters.get('name')
domain_opts = self._api_parameters_to_domain_opts(parameters)
general_opts = self._api_parameters_to_general_opts(parameters)
parameters['id'] = id
domain_opts['name'] = id
domain_opts['phantom_name'] = name
conf = {'engine_conf': domain_opts, 'general': general_opts}
try:
self.epum.reconfigure_domain(id, conf, caller=self.access_key)
return parameters
except Exception:
log.exception("Problem modifying domain: %s" % name)
raise
def get_all_domains(self, username):
domain_names = self.epum.list_domains(caller=self.access_key)
return domain_names
def _sanitize_sensor_data(self, sensor_data):
cleaned_sensor_data = {}
for key, item in sensor_data.iteritems():
key = key.lower()
cleaned_item = {}
for k, v in item.iteritems():
k = k.lower()
cleaned_item[k] = v
cleaned_sensor_data[key] = cleaned_item
return cleaned_sensor_data
def get_domain(self, username, id):
try:
domain_description = self.epum.describe_domain(
id, caller=self.access_key)
except DashiError:
return None
engine_conf = domain_description.get('config',
{}).get('engine_conf', {})
general_conf = domain_description.get('config', {}).get('general', {})
ent = {}
ent['id'] = domain_description['name']
ent['de_name'] = engine_conf.get('phantom_de_name')
ent['name'] = engine_conf.get('phantom_name')
ent['sensor_data'] = self._sanitize_sensor_data(
domain_description.get('sensor_data', {}))
ent['monitor_sensors'] = ",".join(
engine_conf.get('monitor_sensors', []))
ent['monitor_domain_sensors'] = ",".join(
engine_conf.get('monitor_domain_sensors', []))
if 'chef_credential' in general_conf:
ent['chef_credential'] = general_conf['chef_credential']
if ent['de_name'] == 'multicloud':
ent['vm_count'] = engine_conf.get('minimum_vms')
ent['lc_name'] = engine_conf.get('dtname')
elif ent['de_name'] == 'sensor':
ent['lc_name'] = engine_conf.get('dtname')
ent['sensor_minimum_vms'] = engine_conf.get('minimum_vms')
ent['sensor_maximum_vms'] = engine_conf.get('maximum_vms')
ent['sensor_metric'] = engine_conf.get('metric')
ent['sensor_scale_down_threshold'] = engine_conf.get(
'scale_down_threshold')
ent['sensor_scale_down_vms'] = engine_conf.get('scale_down_n_vms')
ent['sensor_scale_up_threshold'] = engine_conf.get(
'scale_up_threshold')
ent['sensor_scale_up_vms'] = engine_conf.get('scale_up_n_vms')
ent['sensor_cooldown'] = engine_conf.get('cooldown_period')
return ent
def get_domain_instances(self, username, id):
try:
domain_description = self.epum.describe_domain(
id, caller=self.access_key)
except DashiError:
return None
instances = domain_description.get('instances', [])
parsed_instances = []
for i in instances:
instance = {
'id': i.get('instance_id', ''),
'iaas_instance_id': i.get('iaas_id', ''),
'lifecycle_state': i.get('state', ''),
'hostname': i.get('hostname', ''),
'public_ip': i.get('public_ip', ''),
'private_ip': i.get('private_ip', ''),
'cloud': i.get('site', ''),
'image_id': i.get('iaas_image', ''),
'instance_type': i.get('iaas_allocation', ''),
'sensor_data':
self._sanitize_sensor_data(i.get('sensor_data', {})),
'keyname': i.get('iaas_sshkeyname', ''),
}
parsed_instances.append(instance)
return parsed_instances
def get_all_groups(self):
domain_names = self.epum.list_domains(caller=self.access_key)
domains = []
for domain in domain_names:
domain_description = self.epum.describe_domain(
domain, caller=self.access_key)
domains.append(domain_description)
return domains
def create_dt(self, dt_name, cloud_params, context_params, appliance=None):
dt = self.get_dt(dt_name)
if dt is None:
dt = {}
dt['mappings'] = {}
create = True
else:
create = False
cloud_credentials = self.get_clouds()
for cloud_name, parameters in cloud_params.iteritems():
mapping = dt['mappings'].get(cloud_name)
if mapping is None:
mapping = dt['mappings'][cloud_name] = {}
credentials = cloud_credentials.get(cloud_name, None)
# Required by EPUM
mapping['iaas_allocation'] = parameters.get('instance_type')
mapping['iaas_image'] = parameters.get('image_id')
if credentials is not None:
mapping['key_name'] = credentials.keyname
else:
# TODO: raise error?
mapping['key_name'] = ''
# Phantom stuff
mapping['common'] = parameters.get('common')
mapping['rank'] = parameters.get('rank')
mapping['max_vms'] = parameters.get('max_vms')
# Contextualization
if context_params.get('contextualization_method'
) == 'user_data' or context_params.get(
'user_data'):
contextualization = dt.get('contextualization')
if contextualization is None:
contextualization = dt['contextualization'] = {}
contextualization['method'] = 'userdata'
contextualization['userdata'] = context_params['user_data']
elif context_params.get('contextualization_method') == 'chef':
contextualization = dt.get('contextualization')
if contextualization is None:
contextualization = dt['contextualization'] = {}
if contextualization.get('userdata') is not None:
del contextualization['userdata']
contextualization['method'] = 'chef'
try:
contextualization['run_list'] = json.loads(
context_params.get('chef_runlist', '[]'))
except Exception:
log.exception("Problem parsing LC content")
raise PhantomWebException(
"Problem parsing runlist when creating LC: %s" %
context_params.get('chef_runlist'))
try:
contextualization['attributes'] = json.loads(
context_params.get('chef_attributes', '{}'))
except Exception:
log.exception("Problem parsing LC content")
raise PhantomWebException(
"Problem parsing chef attributes when creating LC: %s" %
context_params.get('chef_attributes'))
elif context_params.get('contextualization_method') == 'none':
contextualization = dt['contextualization'] = {}
contextualization['method'] = None
if appliance is not None:
dt['appliance'] = appliance
else:
if 'appliance' in dt:
del dt['appliance']
if create:
return self.dtrs.add_dt(self.access_key, dt_name, dt)
else:
return self.dtrs.update_dt(self.access_key, dt_name, dt)
def get_dt(self, dt_name):
return self.dtrs.describe_dt(self.access_key, dt_name)
def remove_dt(self, dt_name):
return self.dtrs.remove_dt(self.access_key, dt_name)
def get_all_lcs(self):
dt_names = self.dtrs.list_dts(self.access_key)
dts = []
for dt_name in dt_names:
dt = self.dtrs.describe_dt(self.access_key, dt_name)
dts.append(dt)
return dts
def _load_clouds(self):
sites = self.dtrs.list_credentials(self.access_key)
self.iaasclouds = {}
for site_name in sites:
try:
site_desc = self.dtrs.describe_site(self.access_key, site_name)
desc = self.dtrs.describe_credentials(self.access_key,
site_name)
uci = UserCloudInfo(site_name, self.username,
desc['access_key'], desc['secret_key'],
desc['key_name'], site_desc)
self.iaasclouds[site_name] = uci
except Exception, ex:
log.error(
"Failed trying to add the site %s to the user %s | %s" %
(site_name, self.username, str(ex)))
def get_possible_sites(self, details=False):
site_client = DTRSClient(self._dashi_conn)
site_names = site_client.list_sites(self.access_key)
all_sites = {}
for site in site_names:
all_sites[site] = {'id': site, 'instance_types': INSTANCE_TYPES}
if details is True:
pool = Pool()
instance_types_results = {}
for site in site_names:
instance_types_results[site] = pool.apply_async(
site_client.describe_site, [self.access_key, site])
pool.close()
for site in site_names:
try:
site_description = instance_types_results[site].get(
IAAS_TIMEOUT)
instance_types = site_description.get("instance_types")
if instance_types is not None:
all_sites[site]['instance_types'] = instance_types
cloud_type = site_description.get("type")
if cloud_type is not None:
all_sites[site]['type'] = cloud_type
image_generation = site_description.get("image_generation")
if image_generation:
all_sites[site]['image_generation'] = True
else:
all_sites[site]['image_generation'] = False
except TimeoutError:
log.exception(
"Timed out getting list of instance types for %s" %
site)
except Exception:
log.exception(
"Unexpected error getting list of instance types for %s"
% site)
pool = Pool()
public_results = {}
user_results = {}
clouds = self.get_clouds()
for cloud_name, cloud in clouds.iteritems():
result = pool.apply_async(cloud.get_user_images)
user_results[cloud_name] = result
if cloud.site_desc["type"] != "ec2":
result = pool.apply_async(cloud.get_public_images)
public_results[cloud_name] = result
pool.close()
for cloud_name, result in user_results.iteritems():
try:
all_sites[cloud_name]['user_images'] = result.get(
IAAS_TIMEOUT)
except TimeoutError:
log.exception("Timed out getting images from %s" %
cloud_name)
all_sites[cloud_name]['user_images'] = []
except Exception:
log.exception("Unexpected error getting images from %s" %
cloud_name)
all_sites[cloud_name]['user_images'] = []
for cloud_name, result in public_results.iteritems():
try:
all_sites[cloud_name]['public_images'] = result.get(
IAAS_TIMEOUT)
except TimeoutError:
log.exception("Timed out getting images from %s" %
cloud_name)
all_sites[cloud_name]['public_images'] = []
except Exception:
log.exception("Unexpected error getting images from %s" %
cloud_name)
all_sites[cloud_name]['public_images'] = []
return all_sites