def creating(self, model: Zone):
defer(model.save)
with self.vmware.client_session() as vmware_client:
datacenter = self.vmware.get_datacenter(vmware_client,
model.region.datacenter)
datastore = self.vmware.get_datastore(vmware_client,
model.vm_datastore,
datacenter)
if datastore is None:
model.error_message = "Could not find VMWare datastore."
return
cluster = self.vmware.get_cluster(vmware_client, model.vm_cluster,
datacenter)
if cluster is None:
model.error_message = "Could not find VMWare cluster."
return
if model.vm_folder is not None:
folder = self.vmware.get_folder(vmware_client, model.vm_folder,
datacenter)
if folder is None:
model.error_message = "Could not find VMWare VM & Templates folder."
return
model.state = ResourceState.Created
def on_stopped_leading(self):
self.logger.info("Stopped leading... stopping controllers")
self.lock.acquire()
defer(self.lock.release)
for controller in self.controllers:
controller.stop()
self.controllers = []
def creating(self, model: Image):
region = model.region
if region.state == ResourceState.Deleting:
model.delete()
return
if model.task == ImageTask.IMAGING_INSTANCE:
if model.file_name is None:
# Image was created via instance so lets wait until the file is ready
from_instance = Instance.get(
model.project, model.task_kwargs['instance_name'])
# If the from_instance is gone the image should be deleted
if from_instance is None:
model.delete()
return
# If the instance errored creating the image we should be deleted
if from_instance.state == ResourceState.Error:
model.delete()
return
return
else:
model.task = None
defer(model.save)
with self.vmware.client_session() as vmware_client:
datacenter = self.vmware.get_datacenter(vmware_client,
region.datacenter)
vmware_image = self.vmware.get_image(vmware_client,
model.file_name, datacenter)
if vmware_image is None:
model.error_message = "Could not find image file"
return
model.state = ResourceState.Created
def done(self, item):
self.condition.acquire()
defer(self.condition.release)
self.processing.remove(item)
if item in self.dirty:
self.queue.append(item)
self.condition.notify()
def main():
print('opening demo-1.txt')
f1 = open('demo-1.txt', 'rt')
defer(do_close, f1)
print('opening demo-2.txt')
f2 = open('demo-2.txt', 'rt')
defer(do_close, f2)
print('returning...')
def deleting(self, model: Volume):
defer(model.save)
if model.backing_id is not None:
with self.vmware.client_session() as vmware_client:
datacenter = self.vmware.get_datacenter(
vmware_client, model.region.datacenter)
datastore = self.vmware.get_datastore(vmware_client,
model.zone.vm_datastore,
datacenter)
self.vmware.delete_disk(vmware_client, model.backing_id,
datastore)
model.state = ResourceState.Deleted
def add(self, item):
self.condition.acquire()
defer(self.condition.release)
if self.__shutting_down:
return
if item in self.dirty:
return
self.dirty.append(item)
if item in self.processing:
return
self.queue.append(item)
self.condition.notify()
def get(self):
self.condition.acquire()
defer(self.condition.release)
while len(self.queue) == 0 and self.__shutting_down is False:
self.condition.wait()
if len(self.queue) == 0:
return None, True
item, self.queue = self.queue[0], self.queue[1:]
self.processing.append(item)
self.dirty.remove(item)
return item, False
def creating(self, model: Volume):
defer(model.save)
region: Region = model.region
if region.schedulable is False:
model.error_message = "Region is not currently schedulable"
return
zone: Zone = model.zone
if zone.schedulable is False:
model.error_message = "Zone is not currently schedulable"
return
with self.vmware.client_session() as vmware_client:
datacenter = self.vmware.get_datacenter(vmware_client,
region.datacenter)
datastore = self.vmware.get_datastore(vmware_client,
zone.vm_datastore,
datacenter)
cloned_from: Volume = model.cloned_from
if cloned_from is None:
if model.cloned_from_name:
model.error_message = "Could not clone volume, parent diapered."
return
model.backing_id = self.vmware.create_disk(
vmware_client, model.name, model.size, datastore)
model.task = None
model.state = ResourceState.Created
return
else:
if 'task_key' not in model.task_kwargs:
task = self.vmware.clone_disk(vmware_client, model.name,
str(cloned_from.backing_id),
datastore)
model.task_kwargs = {"task_key": task.info.key}
else:
task = self.vmware.get_task(vmware_client,
model.task_kwargs['task_key'])
done, error = self.vmware.is_task_done(task)
if done:
if error is not None:
model.error_message = error
return
model.backing_id = task.info.result.config.name.name
model.task = None
model.state = ResourceState.Created
def deleting(self, model):
defer(model.save)
if model.file_name is not None:
# Only try and delete if we have a file
region = model.region
with self.vmware.client_session() as vmware_client:
datacenter = self.vmware.get_datacenter(
vmware_client, region.datacenter)
vmware_image = self.vmware.get_image(vmware_client,
model.file_name,
datacenter)
if vmware_image is not None:
self.vmware.delete_image(vmware_client, vmware_image)
else:
self.logger.warning(
"Tried to delete image %s but couldn't find its backing file"
% str(model.name))
model.state = ResourceState.Deleted
def creating(self, model: NetworkPort):
network: Network = model.network
if network is None:
model.delete()
return
defer(model.save)
usable_addresses = []
start_host = network.pool_start
end_host = network.pool_end
for host in network.cidr:
if host == network.gateway:
# Skip gateway
continue
if host in network.dns_servers:
# Skip dns servers if in range
continue
if start_host <= host <= end_host:
usable_addresses.append(host)
self.lock.acquire()
defer(self.lock.release)
network_ports = NetworkPort.list_all(label_selector=NETWORK_LABEL +
"=" + str(network.name))
for network_port in network_ports:
if network_port.ip_address is not None:
usable_addresses.remove(network_port.ip_address)
if len(usable_addresses) == 0:
model.error_message = "No usable ip addresses found."
return
model.ip_address = usable_addresses[0]
model.state = ResourceState.Created
# We need to save before the lock is released
model.save()
def on_started_leading(self):
if self.leader_elector.shutting_down:
return
self.lock.acquire()
defer(self.lock.release)
self.logger.info("Started leading... starting controllers")
self.launch_controller(RegionController(1, 30, self.vmware))
self.launch_controller(ZoneController(1, 30, self.vmware))
self.launch_controller(IAMSystemRoleController(1, 30))
self.launch_controller(IAMProjectRoleController(1, 30))
self.launch_controller(IAMPolicyController(1, 30))
self.launch_controller(ProjectQuotaController(1, 30))
self.launch_controller(NetworkController(1, 30, self.vmware))
self.launch_controller(NetworkPortController(1, 30))
self.launch_controller(ImageController(4, 30, self.vmware))
self.launch_controller(SystemServiceAccountController(1, 30))
self.launch_controller(ProjectServiceAccountController(1, 30))
self.launch_controller(FlavorController(1, 30))
self.launch_controller(VolumeController(4, 30, self.vmware))
self.launch_controller(InstanceController(4, 30, self.vmware, self.menu_url))
self.launch_controller(KeypairController(4, 30))
def creating(self, model: Network):
defer(model.save)
region = model.region
if region is None:
model.state = ResourceState.ToDelete
return
with self.vmware.client_session() as vmware_client:
datacenter = self.vmware.get_datacenter(vmware_client,
region.datacenter)
if datacenter is None:
model.error_message = "Could not find VMWare Datacenter for region %s " % str(
region.name)
return
port_group = self.vmware.get_port_group(vmware_client,
model.port_group,
datacenter)
if port_group is None:
model.error_message = "Could not find port group"
return
model.state = ResourceState.Created
def deleting(self, model: Instance):
defer(model.save)
region = model.region
if region is not None:
with self.vmware.client_session() as vmware_client:
datacenter = self.vmware.get_datacenter(
vmware_client, region.datacenter)
vmware_vm = self.vmware.get_vm(vmware_client, str(model.vm_id),
datacenter)
if vmware_vm is None:
self.logger.warning(
"Could not find backing vm for instance %s/%s when trying to delete"
% (model.project.name, model.name))
else:
power_state = str(vmware_vm.runtime.powerState)
if power_state == 'poweredOn':
is_shutdown = self.shutdown_vm(vmware_client,
vmware_vm, model)
if is_shutdown is False:
return
for idx, volume_name in enumerate(
model.initial_volumes_status):
delete = model.initial_volumes[idx]['auto_delete']
if delete:
volume: Volume = Volume.get(
model.project, volume_name)
if volume is None:
continue
if volume.state in [
ResourceState.ToDelete,
ResourceState.Deleting,
ResourceState.Deleted
]:
continue
if volume.attached_to_name == model.name:
volume.delete()
attached_volumes = Volume.list(
model.project,
label_selector=ATTACHED_TO_LABEL + "=" +
str(model.name))
if len(attached_volumes) > 0:
for volume in attached_volumes:
if volume.state in [
ResourceState.ToDelete,
ResourceState.Deleting,
ResourceState.Deleted
]:
continue
if volume.task != VolumeTask.DETACHING:
volume.task = VolumeTask.DETACHING
volume.save()
return
self.vmware.delete_vm(vmware_client, vmware_vm)
else:
# The region has poofed so we can't actually delete anything in vmware
pass
network_port = model.network_port
if network_port is not None:
network_port.delete()
model.power_state = VMPowerState.POWERED_OFF
model.state = ResourceState.Deleted
def shutting_down(self):
self.condition.acquire()
defer(self.condition.release)
return self.__shutting_down
def shutdown(self):
self.condition.acquire()
defer(self.condition.release)
self.__shutting_down = True
self.condition.notify_all()
def created(self, model: Volume):
zone = model.zone
needs_save = False
def save():
if needs_save:
model.save(ignore=True)
defer(save)
if model.task is not None:
needs_save = True
with self.vmware.client_session() as vmware_client:
datacenter = self.vmware.get_datacenter(
vmware_client, model.region.datacenter)
datastore = self.vmware.get_datastore(vmware_client,
zone.vm_datastore,
datacenter)
if model.task == VolumeTask.ATTACHING:
instance = Instance.get(model.project,
model.task_kwargs['to'])
if instance is None:
# Attaching to instance doesn't exist
model.task = None
return
if instance.state in [
ResourceState.ToDelete, ResourceState.Deleting,
ResourceState.Deleted, ResourceState.Error
]:
# Attaching to instance is deleting or errored.
model.task = None
return
vm = self.vmware.get_vm(vmware_client, str(instance.name),
datacenter)
if vm is None:
# VM doesn't exist
model.task = None
return
self.vmware.attach_disk(vmware_client, model.backing_id,
datastore, vm)
model.attached_to = instance
model.task = None
elif model.task == VolumeTask.DETACHING:
self.detach_disk(vmware_client, datacenter, model)
model.attached_to = None
model.task = None
elif model.task == VolumeTask.GROWING:
self.vmware.grow_disk(vmware_client, model.backing_id,
model.task_kwargs['size'], datastore)
model.size = model.task_kwargs['size']
model.task = None
elif model.task == VolumeTask.CLONING:
# Check new volume
# If it's none, created or errored then we are done cloning
new_volume = Volume.get(model.project,
model.task_kwargs['volume_name'])
if new_volume is None or new_volume.state in [
ResourceState.Created, ResourceState.Error
]:
model.task = None
if zone.state == ResourceState.Deleting:
model.state = ResourceState.ToDelete
needs_save = True
if model.attached_to_name is not None:
if model.attached_to is None:
model.attached_to = None
needs_save = True
if model.cloned_from_name is not None:
if model.cloned_from is None:
model.cloned_from = None
needs_save = True
def get(self, key):
self.lock.acquire()
defer(self.lock.release)
return self.cache.get(key)
def creating(self, model: Instance):
defer(model.save)
if model.task is None:
region: Region = model.region
if region.schedulable is False:
model.error_message = "Region is not currently schedulable"
return
image = model.image
if image is None:
model.error_message = "Image does not exist"
return
network_port: NetworkPort = model.network_port
if network_port.state == ResourceState.Error:
model.error_message = "Network Port has returned an error"
return
if network_port.state != ResourceState.Created:
# Wait for network port to be ready
return
with self.vmware.client_session() as vmware_client:
datacenter = self.vmware.get_datacenter(
vmware_client, region.datacenter)
zone: Zone = model.zone
if zone is None:
zone = self.find_best_zone(vmware_client, datacenter,
region, model)
# If we cannot find a free zone error
if zone is None:
model.error_message = "Could not find an available zone to launch the instance"
return
model.zone = zone
else:
if zone.schedulable is False:
model.error_message = "Zone is not currently schedulable"
return
if self.can_zone_host(vmware_client, datacenter, zone,
model) is False:
model.error_message = "Requested zone does not have enough resources available."
return
vmware_image = self.vmware.get_image(vmware_client,
image.file_name,
datacenter)
image_size = math.ceil(
self.vmware.get_disk_size(vmware_image) / (1024**3))
if image_size > model.disk:
model.error_message = "Requested image requires a disk size of at least %s GB" % image_size
return
#
# old_vm = self.vmware.get_vm(vmware_client, str(model.vm_id), datacenter)
# if old_vm is not None:
# self.logger.info(
# "A backing for the vm {0} / {1} already exists so it is going to be deleted".format(
# model.project.name,
# model.name))
# self.vmware.power_off_vm(vmware_client, old_vm, hard=True)
# self.vmware.delete_vm(vmware_client, old_vm)
port_group = self.vmware.get_port_group(
vmware_client, network_port.network.port_group, datacenter)
cluster = self.vmware.get_cluster(vmware_client,
zone.vm_cluster, datacenter)
datastore = self.vmware.get_datastore(vmware_client,
zone.vm_datastore,
datacenter)
folder = None
if zone.vm_folder is not None:
folder = self.vmware.get_folder(vmware_client,
zone.vm_folder, datacenter)
create_vm_task = self.vmware.create_vm_from_image(
vm_name="sandwich-" + str(uuid.uuid4()),
image=vmware_image,
datacenter=datacenter,
cluster=cluster,
datastore=datastore,
folder=folder,
port_group=port_group,
vcpus=model.vcpus,
ram=model.ram)
model.task = VMTask.BUILDING
model.task_kwargs = {"task_key": create_vm_task.info.key}
elif model.task == VMTask.BUILDING:
with self.vmware.client_session() as vmware_client:
task = self.vmware.get_task(vmware_client,
model.task_kwargs['task_key'])
done, error = self.vmware.is_task_done(task)
if done:
if error is not None:
model.error_message = error
return
model.vm_id = task.info.result.config.instanceUuid
datacenter = self.vmware.get_datacenter(
vmware_client, model.region.datacenter)
vmware_vm = self.vmware.get_vm(vmware_client,
str(model.vm_id),
datacenter)
self.vmware.resize_root_disk(vmware_client, model.disk,
vmware_vm)
self.vmware.setup_serial_connection(
vmware_client, self.vspc_url, vmware_vm)
if len(model.initial_volumes) > 0:
if len(model.initial_volumes_status) == 0:
initial_volume_names = []
for idx, volume_data in enumerate(
model.initial_volumes):
volume = Volume()
volume.project = model.project
volume.name = model.name + "-" + str(idx)
volume.zone = model.zone
volume.size = volume_data['size']
volume.create()
initial_volume_names.append(volume.name)
model.initial_volumes_status = initial_volume_names
return
else:
attached_vols = 0
for volume_name in model.initial_volumes_status:
volume: Volume = Volume.get(
model.project, volume_name)
if volume is None:
model.error_message = "Volume " + volume.name + \
" has disappeared while trying to attach"
return
if volume.state in [
ResourceState.ToCreate,
ResourceState.Creating
]:
continue
if volume.state != ResourceState.Created:
model.error_message = "Cannot attach volume " + str(
volume.name
) + " while it is in the following state: " + volume.state.value
return
if volume.attached_to_name == model.name:
attached_vols += 1
continue
if volume.attached_to_name is not None and volume.attached_to_name != model.name:
model.error_message = "Volume " + str(
volume.name
) + " has been attached to another instance."
return
if volume.task is None:
volume.attach(model)
volume.save()
else:
model.error_message = "Cannot attach volume" + str(
volume.name
) + " while a task is running on it."
return
if attached_vols != len(
model.initial_volumes_status):
return
self.vmware.power_on_vm(vmware_client, vmware_vm)
model.task = None
model.power_state = VMPowerState.POWERED_ON
model.state = ResourceState.Created
def add(self, key, item):
self.lock.acquire()
defer(self.lock.release)
self.cache[key] = item
def reset(self, new_cache):
self.lock.acquire()
defer(self.lock.release)
self.cache = new_cache
def delete(self, key):
self.lock.acquire()
defer(self.lock.release)
self.cache.pop(key, None)
def _process_item(key):
# Remove from workqueue because we are done with the object
defer(self.workqueue.done, key)
self.sync_handler(key)
def len(self):
self.condition.acquire()
defer(self.condition.release)
return len(self.queue)
