def start_ec2_instances(self, image_id=None, key_name=None, group_name=None, num=1, instance_type=None):
if key_name is None:
key_name = self.config['key_name']
if group_name is None:
group_name = self.config['group_name']
if num is None:
num = 1
if instance_type is None:
instance_type = self.config['default_instance_type']
try:
img = self.conn.get_all_images(image_ids=[image_id])[0]
except IndexError:
raise ProviderException("Could not find image_id={0}".format(image_id))
if img.state != "available":
if img.state != "pending":
raise ProviderException("Image {0} is not available, it's state is {1}.".format(image_id, img.state))
while img.state == "pending":
print "Image {0} has state {1}, waiting {2} seconds for it to become available.".format(image_id, img.state, self.PENDING_IMAGE_WAITTIME)
time.sleep(self.PENDING_IMAGE_WAITTIME)
img.update()
print "Starting {0} EC2 instance(s). This will take a minute...".format(num)
reservation = self.conn.run_instances(image_id, min_count=num, max_count=num, key_name=key_name, security_groups=[group_name], instance_type=instance_type)
instances = reservation.instances
num_instance = len(instances)
num_running = 0
while num_running < num_instance:
num_running = 0
for instance in instances:
instance.update()
if instance.state == 'running':
num_running += 1
if num_running < num_instance:
time.sleep(5)
print "EC2 instances started."
return sorted(instances, key=lambda vm: vm.id)
def get_instance(self, instance_id):
#logging.debug("get_instance(instance_id={0})".format(instance_id))
try:
reservations = self.conn.get_all_reservations(instance_ids=[instance_id])
except EC2ResponseError:
raise ProviderException("instance not found {0}".format(instance_id))
#logging.debug("get_instance() reservations:{0}".format(reservations))
for reservation in reservations:
#logging.debug("get_instance() reservation.instances:{0}".format(reservation.instances))
for instance in reservation.instances:
if instance.id == instance_id:
return instance
raise ProviderException("instance not found {0}".format(instance_id))
def resume_instance(self, instances):
self._connect()
if isinstance(instances, list):
ec2_instances = []
for instance in instances:
ec2_instance = self.ec2.get_instance(instance.provider_instance_identifier)
ec2_instances.append(ec2_instance)
new_ec2_instances = self.ec2.resume_ec2_instances(ec2_instances)
instances_to_update = list(instances)
while len(instances_to_update) > 0:
instance = instances_to_update.pop()
success=False
for ec2_inst in new_ec2_instances:
if ec2_inst.id == instance.provider_instance_identifier:
instance.ip_address = ec2_inst.public_dns_name
logging.debug("instance.id={0} updated with ip={1}".format(instance.provider_instance_identifier, instance.ip_address))
success=True
break
if not success:
raise ProviderException("Could not update the IP of id={0} after resume".format(instance.provider_instance_identifier))
else:
ec2_instance = self.ec2.get_instance(instances.provider_instance_identifier)
new_instance = self.ec2.resume_ec2_instances([ec2_instance])
instances.ip_address = new_instance[0].public_dns_name
logging.debug("instance.id={0} updated with ip={1}".format(instances.provider_instance_identifier, instances.ip_address))
def __init__(self, config=None, connect=True):
if config is not None:
self.config = config
if self.config['aws_access_key'] is None or self.config['aws_secret_key'] is None:
raise ProviderException("AWS_SECRET_KEY or AWS_ACCESS_KEY not set")
if connect:
self.connect()
def _terminate_instances(self, instance_ids):
self._connect()
if not isinstance(instance_ids, list):
instance_ids = [instance_ids]
try:
instances = []
for instance_id in instance_ids:
instance = self.nova.servers.get(instance_id)
instances.append(instance)
instance.delete()
inst_to_check = instances
while len(inst_to_check) > 0:
time.sleep(5)
inst_still_stopping = []
for instance in inst_to_check:
# Retrieve the instance again so the status field updates
try:
instance = self.nova.servers.get(instance.id)
logging.debug("Terminating node, status '{0}' [{1}]".format(instance.status, instance.id))
if instance.status != 'SHUTOFF':
inst_still_stopping.append(instance)
except novaclient.exceptions.NotFound as e:
pass
inst_to_check = inst_still_stopping
except Exception as e:
logging.exception(e)
raise ProviderException("Failed to terminate vm(s)\n{0}".format(e))
def create_molns_image(self):
""" Create the molns image is created. """
self._connect()
# start vm
instances = self.ec2.start_ec2_instances(image_id=self.config["ubuntu_image_name"])
instance = instances[0]
# get login ip
ip = instance.public_dns_name
# install software
try:
logging.debug("installing software on server (ip={0})".format(ip))
install_vm_instance = installSoftware.InstallSW(ip, config=self)
install_vm_instance.run_with_logging()
# create image
logging.debug("shutting down instance")
self.ec2.stop_ec2_instances([instance])
logging.debug("creating image")
image_id = instance.create_image(name=self._get_image_name())
except Exception as e:
logging.exception(e)
raise ProviderException("Failed to create molns image: {0}".format(e))
finally:
logging.debug("terminating {0}".format(instance))
self.ec2.terminate_ec2_instances([instance])
return image_id
def start_instance(self, num=1):
""" Start worker group vms. """
try:
self._connect()
instances = self.eucalyptus.start_eucalyptus_instances(
image_id=self.provider.config["molns_image_name"],
num=int(num),
instance_type=self.config["instance_type"])
ret = []
for instance in instances:
ip = instance.public_dns_name
i = self.datastore.get_instance(
provider_instance_identifier=instance.id,
ip_address=ip,
provider_id=self.provider.id,
controller_id=self.controller.id,
worker_group_id=self.id)
ret.append(i)
if num == 1:
return ret[0]
else:
return ret
except Exception as e:
logging.exception(e)
raise ProviderException(
"Failed to start molns instance: {0}".format(e))
def _attach_floating_ip(self, instance):
# Try to attach a floating IP to the controller
logging.info("Attaching floating ip to the server...")
try:
floating_ip = self.nova.floating_ips.create(self.config['floating_ip_pool'])
instance.add_floating_ip(floating_ip)
logging.debug("ip={0}".format(floating_ip.ip))
return floating_ip.ip
except Exception as e:
raise ProviderException("Failed to attach a floating IP to the controller.\n{0}".format(e))
def _delete_floating_ip(self, ip):
try:
floating_ips = self.nova.floating_ips.list()
for fip in floating_ips:
if fip.ip == ip:
fip.delete()
return
except Exception as e:
logging.exception(e)
raise ProviderException("Could not delete floating ip '{0}'".format(ip))
def get_instance_status(self, instance):
try:
status = self.provider._get_instance_status(instance.provider_instance_identifier)
except novaclient.exceptions.NotFound as e:
return self.STATUS_TERMINATED
if status == 'ACTIVE' or status == 'BUILD':
return self.STATUS_RUNNING
if status == 'SHUTOFF':
return self.STATUS_STOPPED
if status == 'DELETED':
return self.STATUS_TERMINATED
raise ProviderException("OpenStackController.get_instance_status() got unknown status '{0}'".format(status))
def create_security_group(self, group_name):
security_group = None
for sg in self.conn.get_all_security_groups():
if sg.name == group_name:
security_group = sg
break
if security_group is None:
print "Security group not found, creating one."
security_group = self.conn.create_security_group(group_name, 'MOLNs Security Group')
self.set_security_group_rules(security_group)
elif not self.check_security_group_rules(security_group):
raise ProviderException("Security group {0} exists, but has the wrong firewall rules. Please delete the group, or choose a different one.")
return security_group
def create_vm_image(self, image_name=None, key_name=None):
if key_name is None:
key_name = self.config['key_name']
if image_name is None:
image_name = "MOLNS_{0}_{1}".format(key_name,int(time.time()))
(stopped_vms, running_vms) = self.get_vm_status(key_name)
if len(running_vms) != 1:
raise ProviderException("Expected only one running vm, {0} are running".format(len(running_vms)))
self.stop_ec2_instances(running_vms)
instance = running_vms[0]
image_ami = instance.create_image(image_name)
print "Image created id={0} name={0}".format(image_ami, image_name)
self.terminate_ec2_instances(running_vms)
return image_ami
def get_instance_status(self, instance):
self._connect()
try:
status = self.ec2.get_instance_status(instance.provider_instance_identifier)
except Exception as e:
#logging.exception(e)
return self.STATUS_TERMINATED
if status == 'running' or status == 'pending':
return self.STATUS_RUNNING
if status == 'stopped' or status == 'stopping':
return self.STATUS_STOPPED
if status == 'terminated' or status == 'shutting-down':
return self.STATUS_TERMINATED
raise ProviderException("EC2Controller.get_instance_status() got unknown status '{0}'".format(status))
def create_molns_image(self):
""" Create a molns image, save it on localhost and return DockerImage ID of created image. """
file_to_remove = None
try:
dockerfile, file_to_remove = self._create_dockerfile(
installSoftware.InstallSW.get_command_list())
image_id = self.docker.build_image(dockerfile)
return image_id
except Exception as e:
logging.exception(e)
raise ProviderException(
"Failed to create molns image: {0}".format(e))
finally:
if file_to_remove is not None:
os.remove(file_to_remove)
def create_ssh_key(self):
""" Create the ssh key and write the file locally. """
ssh_key_dir = os.path.join(self.config_dir, self.name)
logging.debug('ssh_key_dir={0}'.format(ssh_key_dir))
if not os.path.isdir(ssh_key_dir):
logging.debug('making ssh_key_dir={0}'.format(ssh_key_dir))
os.makedirs(ssh_key_dir)
ssh_key_file = os.path.join(ssh_key_dir,self.config['key_name']+self.SSH_KEY_EXTENSION)
if os.path.isfile(ssh_key_file):
raise ProviderException("ssh_key_file '{0}' already exists".format(self.config['key_name']))
self._connect()
new_key = self.nova.keypairs.create(name=self.config['key_name'])
#with open(ssh_key_file, 'w') as fd:
with os.fdopen(os.open(ssh_key_file, os.O_WRONLY | os.O_CREAT, 0600), 'w') as fd:
fd.write(new_key.private_key)
def create_molns_image(self):
""" Create the molns image is created. """
self._connect()
# clear the network-related persisent udev rules:
#echo "" > /etc/udev/rules.d/70-persistent-net.rules
#echo "" > /lib/udev/rules.d/75-persistent-net-generator.rules
#
# start vm
instances = self.eucalyptus.start_eucalyptus_instances(
image_id=self.config["ubuntu_image_name"])
instance = instances[0]
# get login ip
ip = instance.public_dns_name
# install software
try:
logging.debug("installing software on server (ip={0})".format(ip))
install_vm_instance = installSoftware.InstallSW(ip, config=self)
#install_vm_instance.run_with_logging()
# create image
logging.debug("Shutting down instance")
self.eucalyptus.stop_eucalyptus_instances([instance])
logging.debug("Creating image")
image_id = instance.create_image(name=self._get_image_name())
#logging.debug("Finding volume of instance")
#vol = None
#for v in self.eucalyptus.conn.get_all_volumes():
# if v.attach_data is not None and v.attach_data.instance_id == instance.id:
# vol = v
# break
#if vol is None:
# raise Exception("Can not find volume associated with instance. Base image must be an EBS backed image.")
#snap = vol.create_snapshot()
#logging.debug('Snapshot {0} of volume {1}'.format(snap.id, vol.id))
#image_id = self.eucalyptus.conn.register_image(name=self._get_image_name(), snapshot_id=snap.id, delete_root_volume_on_termination=True)
##deleteOnTermination
#image_id = self.eucalyptus.conn.register_image(name=self._get_image_name(), snapshot_id=snap.id)
logging.debug("Image created: {0}".format(image_id))
except Exception as e:
logging.exception(e)
raise ProviderException(
"Failed to create molns image: {0}".format(e))
finally:
logging.debug("terminating {0}".format(instance))
self.eucalyptus.terminate_eucalyptus_instances([instance])
return image_id
def _resume_instances(self, instance_ids):
self._connect()
for instance_id in instance_ids:
try:
instance = self.nova.servers.get(instance_id)
instance.start()
logging.debug("instance={0}".format(instance))
# wait for boot to complete
while instance.status == 'BUILD':
logging.debug("Launching node, status '{0}'".format(instance.status))
time.sleep(5)
# Retrieve the instance again so the status field updates
instance = self.nova.servers.get(instance.id)
#instance.update()
logging.debug("status: {0}".format(instance.status))
return instance
except Exception as e:
logging.exception(e)
logging.debug("terminating instance {0}".format(instance))
instance.delete()
raise ProviderException("Failed to boot vm\n{0}".format(e))
def create_molns_image(self):
""" Create the molns image is created. """
# start vm
instance = self._boot_ubuntu_vm()
# get login ip
ip = self._attach_floating_ip(instance)
# install software
try:
logging.debug("installing software on server (ip={0})".format(ip))
install_vm_instance = installSoftware.InstallSW(ip, config=self)
install_vm_instance.run_with_logging()
# create image
logging.debug("shutting down instance")
self._stop_vm(instance)
logging.debug("creating image")
image_id = instance.create_image(image_name=self._get_image_name())
logging.debug("image creation started image_id={0}".format(image_id))
time.sleep(10)
t1 = time.time()
while time.time() < t1+self.MAX_IMAGE_CREATION_WAITTIME:
logging.debug("waiting for image creation to complete")
time.sleep(30)
img_status = self._get_image_status(image_id)
logging.debug("image status is {0}".format(img_status))
if img_status != 'SAVING':
break
except Exception as e:
logging.exception(e)
raise ProviderException("Failed to create molns image: {0}".format(e))
finally:
logging.debug("terminating {0}".format(instance))
instance.delete()
try:
logging.debug("deleteing floating ip {0}".format(ip))
self._delete_floating_ip(ip)
except ProviderException as e:
logging.error("Error deleteing floating IP: {0}".format(e))
return image_id
def __boot_vm(self, image_name, instance_type, num=1):
self._connect()
instances = []
try:
image = self.nova.images.get(image_name)
#logging.debug("image={0}".format(image))
flavor = self.nova.flavors.find(name=instance_type)
#logging.debug("flavor={0}".format(flavor))
for n in range(int(num)):
if 'neutron_nic' in self.config and self.config['neutron_nic'] != '':
inst = self.nova.servers.create(name="molns_vm_"+self.name, image=image, flavor=flavor, key_name=self.config["key_name"], security_groups=[self.config["group_name"]],nics=[{'net-id':self.config['neutron_nic']}])
else:
inst = self.nova.servers.create(name="molns_vm_"+self.name, image=image, flavor=flavor, key_name=self.config["key_name"], security_groups=[self.config["group_name"]])
instances.append(inst)
#logging.debug("instance={0}".format(inst))
# wait for boot to complete
inst_to_check = instances
while len(inst_to_check) > 0:
time.sleep(5)
inst_still_building = []
for instance in inst_to_check:
# Retrieve the instance again so the status field updates
instance = self.nova.servers.get(instance.id)
logging.debug("Launching node, status '{0}' [{1}]".format(instance.status, instance.id))
if instance.status == 'BUILD':
inst_still_building.append(instance)
inst_to_check = inst_still_building
if num == 1:
return instances[0]
else:
return instances
except Exception as e:
logging.exception(e)
for instance in instances:
logging.debug("terminating instance {0}".format(instance))
instance.delete()
raise ProviderException("Failed to boot vm\n{0}".format(e))
def start_vms(self, image_id=None, key_name=None, group_name=None, num=None, instance_type=None):
if key_name is None:
key_name = self.config['key_name']
if group_name is None:
group_name = self.config['group_name']
if num is None:
num = 1
if instance_type is None:
instance_type = self.config['default_instance_type']
# Check the group
self.create_security_group(group_name)
#(stopped_vms, running_vms) = self.get_vm_status(key_name)
#if len(running_vms) > 0:
# msg = "Error: {0} VMs are already running with key_name={1}".format(len(running_vms),
# key_name)
# print msg
# raise ProviderException(msg)
if len(stopped_vms) > 0:
return self.resume_ec2_instances(stopped_vms)
if image_id is None:
raise ProviderException("Base Ubuntu image not specified.")
else:
self.image_id = image_id
# Check image
try:
img = self.conn.get_all_images(image_ids=[self.image_id])[0]
except IndexError:
raise ProviderException("Could not find image_id={0}".format(self.image_id))
if img.state != "available":
if img.state != "pending":
raise ProviderException("Image {0} is not available, it has state is {1}.".format(self.image_id, img.state))
while img.state == "pending":
print "Image {0} has state {1}, waiting {2} seconds for it to become available.".format(self.image_id, img.state, self.PENDING_IMAGE_WAITTIME)
time.sleep(self.PENDING_IMAGE_WAITTIME)
img.update()
self.key_name = key_name
self.group_name = group_name
group_list = []
for _ in range(num):
group_list.append(group_name)
print "Starting {0} EC2 instance(s). This will take a minute...".format(num)
reservation = self.conn.run_instances(self.image_id, min_count=num, max_count=num, key_name=key_name, security_groups=group_list, instance_type=instance_type)
instances = reservation.instances
num_instance = len(instances)
num_running = 0
while num_running < num_instance:
num_running = 0
for instance in instances:
instance.update()
if instance.state == 'running':
num_running += 1
if num_running < num_instance:
time.sleep(5)
print "EC2 instances started."
return sorted(instances, key=lambda vm: vm.id)