def __init__(self, username):
self.username = username
phantom_user = PhantomUser.objects.get(username=username)
if phantom_user is None:
msg = 'The user %s is not associated with an access key ID. Please contact your sysadmin' % (
username)
raise PhantomWebException(msg)
self.access_key = phantom_user.access_key_id
rabbit_info_objects = RabbitInfoDB.objects.all()
if not rabbit_info_objects:
raise PhantomWebException(
'The service is mis-configured. Please contact your sysadmin')
self.rabbit_info = rabbit_info_objects[0]
ssl = self.rabbit_info.rabbitssl
self._dashi_conn = DashiCeiConnection(
self.rabbit_info.rabbithost,
self.rabbit_info.rabbituser,
self.rabbit_info.rabbitpassword,
exchange=self.rabbit_info.rabbitexchange,
timeout=60,
port=self.rabbit_info.rabbitport,
ssl=ssl)
self.epum = EPUMClient(self._dashi_conn)
self.dtrs = DTRSClient(self._dashi_conn)
def _api_parameters_to_domain_opts(self, parameters):
domain_opts = {}
de_name = parameters.get('de_name')
domain_opts['phantom_de_name'] = de_name
domain_opts['clouds'] = parameters.get('clouds', [])
if de_name == 'sensor':
try:
domain_opts['dtname'] = parameters['lc_name']
domain_opts['cooldown_period'] = parameters['sensor_cooldown']
domain_opts['maximum_vms'] = parameters['sensor_maximum_vms']
domain_opts['minimum_vms'] = parameters['sensor_minimum_vms']
domain_opts['metric'] = parameters['sensor_metric']
domain_opts['monitor_domain_sensors'] = parameters.get(
'monitor_domain_sensors', '').split(',')
domain_opts['monitor_sensors'] = parameters.get(
'monitor_sensors', '').split(',')
domain_opts['scale_down_n_vms'] = parameters[
'sensor_scale_down_vms']
domain_opts['scale_down_threshold'] = parameters[
'sensor_scale_down_threshold']
domain_opts['scale_up_n_vms'] = parameters[
'sensor_scale_up_vms']
domain_opts['scale_up_threshold'] = parameters[
'sensor_scale_up_threshold']
domain_opts[
'sample_function'] = 'Average' # TODO: make configurable
domain_opts[
'sensor_type'] = 'opentsdb' # TODO: make configurable
domain_opts['opentsdb_port'] = int(settings.OPENTSDB_PORT)
domain_opts['opentsdb_host'] = settings.OPENTSDB_HOST
except KeyError as k:
raise PhantomWebException(
"Mandatory parameter '%s' is missing" % k.args[0])
elif de_name == 'multicloud':
try:
domain_opts['dtname'] = parameters['lc_name']
domain_opts['maximum_vms'] = parameters['vm_count']
domain_opts['minimum_vms'] = parameters['vm_count']
domain_opts['monitor_domain_sensors'] = parameters.get(
'monitor_domain_sensors', '').split(',')
domain_opts['monitor_sensors'] = parameters.get(
'monitor_sensors', '').split(',')
domain_opts[
'sample_function'] = 'Average' # TODO: make configurable
domain_opts[
'sensor_type'] = 'opentsdb' # TODO: make configurable
domain_opts['opentsdb_port'] = int(settings.OPENTSDB_PORT)
domain_opts['opentsdb_host'] = settings.OPENTSDB_HOST
except KeyError as k:
raise PhantomWebException(
"Mandatory parameter '%s' is missing" % k.args[0])
else:
raise PhantomWebException("de_name '%s' is not supported" %
de_name)
return domain_opts
def get_image_build(username, image_build_id):
try:
image_build = ImageBuild.objects.get(id=image_build_id, owner=username)
except ImageBuild.DoesNotExist:
raise PhantomWebException(
"Could not find image build %s. Doesn't exist." % image_build_id)
ret = {
"id": image_build.id,
"owner": username,
"cloud_name": image_build.cloud_name
}
if image_build.status == "successful":
ret["ready"] = True
elif image_build.status == "submitted":
result = AsyncResult(image_build.celery_task_id)
ready = result.ready()
ret["ready"] = ready
if ready:
if result.successful():
image_build.returncode = result.result["returncode"]
if image_build.returncode == 0:
image_build.status = "successful"
else:
image_build.status = "failed"
for cloud_name in result.result["artifacts"]:
image_build_artifact = ImageBuildArtifact.objects.create(
image_build_id=image_build.id,
cloud_name=cloud_name,
image_name=result.result["artifacts"][cloud_name])
image_build_artifact.save()
image_build.full_output = result.result["full_output"]
image_build.save()
else:
image_build.status = "failed"
image_build.returncode = -1
image_build.full_output = str(result.result)
image_build.save()
ret["status"] = image_build.status
if image_build.status != "submitted":
ret["returncode"] = image_build.returncode
ret["full_output"] = image_build.full_output
ret["artifacts"] = {}
try:
artifacts = ImageBuildArtifact.objects.filter(
image_build_id=image_build_id)
for artifact in artifacts:
ret["artifacts"][artifact.cloud_name] = artifact.image_name
except ImageBuildArtifact.DoesNotExist:
raise PhantomWebException(
"Could not find image build artifact for image build id %s. Doesn't exist."
% image_build_id)
return ret
def upload_key(self, key_name, key):
connection = self.get_iaas_compute_con()
try:
connection.import_key_pair(key_name, key)
except Exception, ex:
raise PhantomWebException("Error uploading ssh key to %s: %s" %
(key_name, ex))
def _connect_euca(self):
if 'host' and 'port' not in self.site_desc:
raise PhantomWebException("The site %s is misconfigured." %
(self.cloudname))
if self.site_desc['secure']:
scheme = "https"
else:
scheme = "http"
site_url = "%s://%s:%s" % (scheme, self.site_desc['host'],
str(self.site_desc['port']))
kwargs = {}
uparts = urlparse.urlparse(site_url)
is_secure = uparts.scheme == 'https'
if self.site_desc.get('path') is not None:
kwargs['path'] = self.site_desc['path']
ec2conn = EC2Connection(self.iaas_key,
self.iaas_secret,
host=uparts.hostname,
port=uparts.port,
is_secure=is_secure,
validate_certs=False,
**kwargs)
ec2conn.host = uparts.hostname
return ec2conn
def get_image_generator(id):
try:
image_generator = ImageGenerator.objects.get(id=id)
except ImageGenerator.DoesNotExist:
return None
image_generator_dict = {
"id": image_generator.id,
"name": image_generator.name,
"owner": image_generator.username,
"cloud_params": {},
"script": None,
}
cloud_configs = image_generator.imagegeneratorcloudconfig_set.all()
for cc in cloud_configs:
cloud_name = cc.cloud_name
image_generator_dict["cloud_params"][cloud_name] = {
"image_id": cc.image_name,
"ssh_username": cc.ssh_username,
"instance_type": cc.instance_type,
"common": cc.common_image,
"new_image_name": cc.new_image_name,
"public_image": cc.public_image
}
scripts = image_generator.imagegeneratorscript_set.all()
if len(scripts) != 1:
raise PhantomWebException("There should be only 1 script, not %d" %
len(scripts))
image_generator_dict["script"] = scripts[0].script_content
return image_generator_dict
def get_cloud(self, name):
self._load_clouds()
if name not in self.iaasclouds:
raise PhantomWebException(
"No cloud named %s associated with the user. You have %s" %
(name, self.iaasclouds))
return self.iaasclouds[name]
def remove_image_generator(id):
try:
image_generator = ImageGenerator.objects.get(id=id)
except ImageGenerator.DoesNotExist:
raise PhantomWebException(
"Could not delete image generator %s. Doesn't exist." % id)
image_generator.delete()
def remove_image_build(username, image_build_id):
try:
image_build = ImageBuild.objects.get(id=image_build_id, owner=username)
except ImageBuild.DoesNotExist:
raise PhantomWebException(
"Could not find image build %s. Doesn't exist." % image_build_id)
image_build.delete()
def get_iaas_compute_con(self):
if self.site_desc['type'] == "nimbus":
return self._connect_nimbus()
elif self.site_desc['type'] == "ec2":
return self._connect_ec2()
elif self.site_desc['type'] == "openstack":
return self._connect_euca()
raise PhantomWebException("Unknown site type")
def delete_chef_credentials(self, name):
credential_names = self.dtrs.list_credentials(self.access_key,
credential_type="chef")
if name not in credential_names:
raise PhantomWebException("Unknown credentials %s" % name)
return self.dtrs.remove_credentials(self.access_key,
name,
credential_type='chef')
def remove_launch_configuration(username, lc_id):
try:
lc = LaunchConfiguration.objects.get(id=lc_id)
except LaunchConfiguration.DoesNotExist:
raise PhantomWebException(
"Could not delete launch configuration %s. Doesn't exist." % lc_id)
user_obj = get_user_object(lc.username)
try:
user_obj.remove_dt(lc.name)
except Exception:
log.exception("Couldn't delete dt %s" % lc.name)
lc.delete()
def terminate_domain_instance(username, domain_id, instance_id):
user_obj = get_user_object(username)
instance_to_terminate = get_domain_instance(username, domain_id,
instance_id)
if instance_to_terminate is None:
raise PhantomWebException("No instance %s available to terminate" %
instance_id)
instance_iaas_id = instance_to_terminate.get('iaas_instance_id')
if instance_iaas_id is None:
raise PhantomWebException("Instance %s has no iaas ID" % instance_id)
cloud_name = instance_to_terminate.get('cloud')
cloud_name = cloud_name.split("/")[-1]
iaas_cloud = user_obj.get_cloud(cloud_name)
iaas_connection = iaas_cloud.get_iaas_compute_con()
log.debug("User %s terminating the instance %s on %s" %
(username, instance_iaas_id, cloud_name))
timer = statsd.Timer('phantomweb')
timer.start()
timer_cloud = statsd.Timer('phantomweb')
timer_cloud.start()
try:
iaas_connection.terminate_instances(instance_ids=[
instance_iaas_id,
])
except Exception:
log.exception("Couldn't terminate %s" % instance_iaas_id)
timer.stop('terminate_instances.timing')
timer_cloud.stop('terminate_instances.%s.timing' % cloud_name)
return
def update_launch_configuration(id,
cloud_params,
context_params,
appliance=None):
lc = get_launch_configuration(id)
if lc is None:
raise PhantomWebException(
"Trying to update lc %s that doesn't exist?" % id)
username = lc.get('owner')
name = lc.get('name')
user_obj = get_user_object(username)
user_obj.create_dt(name, cloud_params, context_params, appliance)
return lc
def create_domain(username, name, parameters):
user_obj = get_user_object(username)
lc_name = parameters.get('lc_name')
lc = get_launch_configuration_by_name(username, lc_name)
if lc is None:
raise PhantomWebException(
"No launch configuration named %s. Can't make domain" % lc_name)
lc_dict = get_launch_configuration(lc.id)
clouds = []
for cloud_name, cloud in lc_dict.get('cloud_params', {}).iteritems():
cloud = {
'site_name': cloud_name,
'rank': cloud.get('rank'),
'size': cloud.get('max_vms'),
}
clouds.append(cloud)
parameters['clouds'] = clouds
return user_obj.add_domain(username, name, parameters)
def create_image_generator(username, name, cloud_params, script):
image_generator = ImageGenerator.objects.create(name=name,
username=username)
image_generator.save()
for cloud_name in cloud_params:
params = cloud_params[cloud_name]
image_name = params.get("image_id")
instance_type = params.get("instance_type")
ssh_username = params.get("ssh_username")
common_image = params.get("common")
new_image_name = params.get("new_image_name")
public_image = params.get("public_image")
if image_name is None:
raise PhantomWebException(
"You must provide an image_id in the cloud parameters")
if instance_type is None:
raise PhantomWebException(
"You must provide an instance_type in the cloud parameters")
if ssh_username is None:
raise PhantomWebException(
"You must provide an ssh_username in the cloud parameters")
if common_image is None:
raise PhantomWebException(
"You must provide a common boolean in the cloud parameters")
if new_image_name is None:
raise PhantomWebException(
"You must provide a new_image_name in the cloud parameters")
if public_image is None:
raise PhantomWebException(
"You must provide a public_image boolean in the cloud parameters"
)
igcc = ImageGeneratorCloudConfig.objects.create(
image_generator=image_generator,
cloud_name=cloud_name,
image_name=image_name,
ssh_username=ssh_username,
instance_type=instance_type,
common_image=common_image,
new_image_name=new_image_name,
public_image=public_image)
igcc.save()
igs = ImageGeneratorScript.objects.create(
image_generator=image_generator, script_content=script)
igs.save()
return image_generator
def _get_launch_configuration(phantom_con, lc_db_object):
"""
we are only dealing with launch configurations that were made with the web app
"""
lc_name = lc_db_object.name
site_dict = {}
host_vm_db_objs_a = HostMaxPairDB.objects.filter(
launch_config=lc_db_object)
for host_vm_db_obj in host_vm_db_objs_a:
site_name = host_vm_db_obj.cloud_name
shoe_horn_lc_name = "%s@%s" % (lc_name, site_name)
try:
lcs = phantom_con.get_all_launch_configurations(names=[
shoe_horn_lc_name,
])
except Exception, ex:
raise PhantomWebException(
"Error communicating with Phantom REST. %s might be misconfigured | %s"
% (shoe_horn_lc_name, str(ex)))
if len(lcs) != 1:
log.error(
"Received empty launch configuration list from Phantom REST. %s might be misconfigured"
% (shoe_horn_lc_name, ))
continue
lc = lcs[0]
site_entry = {
'cloud': site_name,
'image_id': lc.image_id,
'instance_type': lc.instance_type,
'keyname': lc.key_name,
'user_data': lc.user_data,
'common': host_vm_db_obj.common_image,
'max_vm': host_vm_db_obj.max_vms,
'rank': host_vm_db_obj.rank
}
site_dict[site_name] = site_entry
def create_dt(self, dt_name, cloud_params, context_params, appliance=None):
dt = self.get_dt(dt_name)
if dt is None:
dt = {}
dt['mappings'] = {}
create = True
else:
create = False
cloud_credentials = self.get_clouds()
for cloud_name, parameters in cloud_params.iteritems():
mapping = dt['mappings'].get(cloud_name)
if mapping is None:
mapping = dt['mappings'][cloud_name] = {}
credentials = cloud_credentials.get(cloud_name, None)
# Required by EPUM
mapping['iaas_allocation'] = parameters.get('instance_type')
mapping['iaas_image'] = parameters.get('image_id')
if credentials is not None:
mapping['key_name'] = credentials.keyname
else:
# TODO: raise error?
mapping['key_name'] = ''
# Phantom stuff
mapping['common'] = parameters.get('common')
mapping['rank'] = parameters.get('rank')
mapping['max_vms'] = parameters.get('max_vms')
# Contextualization
if context_params.get('contextualization_method'
) == 'user_data' or context_params.get(
'user_data'):
contextualization = dt.get('contextualization')
if contextualization is None:
contextualization = dt['contextualization'] = {}
contextualization['method'] = 'userdata'
contextualization['userdata'] = context_params['user_data']
elif context_params.get('contextualization_method') == 'chef':
contextualization = dt.get('contextualization')
if contextualization is None:
contextualization = dt['contextualization'] = {}
if contextualization.get('userdata') is not None:
del contextualization['userdata']
contextualization['method'] = 'chef'
try:
contextualization['run_list'] = json.loads(
context_params.get('chef_runlist', '[]'))
except Exception:
log.exception("Problem parsing LC content")
raise PhantomWebException(
"Problem parsing runlist when creating LC: %s" %
context_params.get('chef_runlist'))
try:
contextualization['attributes'] = json.loads(
context_params.get('chef_attributes', '{}'))
except Exception:
log.exception("Problem parsing LC content")
raise PhantomWebException(
"Problem parsing chef attributes when creating LC: %s" %
context_params.get('chef_attributes'))
elif context_params.get('contextualization_method') == 'none':
contextualization = dt['contextualization'] = {}
contextualization['method'] = None
if appliance is not None:
dt['appliance'] = appliance
else:
if 'appliance' in dt:
del dt['appliance']
if create:
return self.dtrs.add_dt(self.access_key, dt_name, dt)
else:
return self.dtrs.update_dt(self.access_key, dt_name, dt)
def modify_image_generator(id, image_generator_params):
try:
image_generator = ImageGenerator.objects.get(id=id)
except ImageGenerator.DoesNotExist:
raise PhantomWebException(
"Trying to update image generator %s that doesn't exist?" % id)
# Required params: name, script, cloud_params
name = image_generator_params.get("name")
if name is None:
raise PhantomWebException(
"Must provide 'name' element to update an image generator")
script = image_generator_params.get("script")
if script is None:
raise PhantomWebException(
"Must provide 'script' element to update an image generator")
cloud_params = image_generator_params.get("cloud_params")
if cloud_params is None:
raise PhantomWebException(
"Must provide 'cloud_params' element to update an image generator")
image_generator.name = name
image_generator.save()
scripts = image_generator.imagegeneratorscript_set.all()
if len(scripts) != 1:
raise PhantomWebException("There should be only 1 script, not %d" %
len(scripts))
scripts[0].script_content = script
scripts[0].save()
cloud_configs = image_generator.imagegeneratorcloudconfig_set.all()
for cc in cloud_configs:
cc.delete()
for cloud_name in cloud_params:
params = cloud_params[cloud_name]
image_name = params.get("image_id")
instance_type = params.get("instance_type")
ssh_username = params.get("ssh_username")
common_image = params.get("common")
new_image_name = params.get("new_image_name")
if image_name is None:
raise PhantomWebException(
"You must provide an image_id in the cloud parameters")
if instance_type is None:
raise PhantomWebException(
"You must provide an instance_type in the cloud parameters")
if ssh_username is None:
raise PhantomWebException(
"You must provide an ssh_username in the cloud parameters")
if common_image is None:
raise PhantomWebException(
"You must provide a common boolean in the cloud parameters")
if new_image_name is None:
raise PhantomWebException(
"You must provide a new_image_name in the cloud parameters")
igcc = ImageGeneratorCloudConfig.objects.create(
image_generator=image_generator,
cloud_name=cloud_name,
image_name=image_name,
ssh_username=ssh_username,
instance_type=instance_type,
common_image=common_image,
new_image_name=new_image_name)
igcc.save()
return get_image_generator(id)
def create_image_build(username, image_generator, additional_credentials={}):
user_obj = get_user_object(username)
all_clouds = user_obj.get_clouds()
sites = {}
credentials = {}
for site in image_generator["cloud_params"]:
try:
cloud = all_clouds[site]
sites[site] = cloud.site_desc
credentials[site] = {
"access_key": cloud.iaas_key,
"secret_key": cloud.iaas_secret,
}
if sites[site]["type"] == "nimbus":
try:
packer_credentials = PackerCredential.objects.get(
username=username, cloud=site)
credentials[site][
"canonical_id"] = packer_credentials.canonical_id
credentials[site][
"usercert"] = packer_credentials.certificate
credentials[site]["userkey"] = packer_credentials.key
except PackerCredential.DoesNotExist:
raise PhantomWebException(
"Could not find extra Nimbus credentials for image generation."
)
elif sites[site]["type"] == "openstack":
try:
packer_credentials = PackerCredential.objects.get(
username=username, cloud=site)
credentials[site][
"openstack_username"] = packer_credentials.openstack_username
credentials[site][
"openstack_password"] = packer_credentials.openstack_password
credentials[site][
"openstack_project"] = packer_credentials.openstack_project
except PackerCredential.DoesNotExist:
raise PhantomWebException(
"Could not find extra OpenStack credentials for image generation."
)
if site in additional_credentials:
openstack_password = additional_credentials[site].get(
"openstack_password")
if openstack_password is not None:
credentials[site][
"openstack_password"] = openstack_password
except KeyError:
raise PhantomWebException("Could not get cloud %s" % site)
result = packer_build.delay(image_generator, sites, credentials)
image_build = ImageBuild.objects.create(
image_generator_id=image_generator["id"],
celery_task_id=result.id,
status='submitted',
returncode=-1,
full_output="",
cloud_name=site,
owner=username)
image_build.save()
return {"id": image_build.id, "ready": result.ready(), "owner": username}