def get(self, network, service_name):
"""
Get a service in "network" named "service_name".
"""
logger.debug('Discovering service %s, %s', network.name, service_name)
# 1. Get list of instances
instances = []
node_name = "%s-%s" % (network.name, service_name)
for node in self.driver.list_nodes():
if node.name.startswith(node_name):
instances.append(canonicalize_instance_info(node))
# 2. Get List Of Subnets
subnetworks = self.subnetwork.get(network, service_name)
if not subnetworks:
return None
# 3. Group Services By Subnet
for subnet_info in subnetworks:
for instance in instances:
if (instance.private_ip and ipaddress.IPv4Network(
subnet_info.cidr_block).overlaps(
ipaddress.IPv4Network(instance.private_ip))):
subnet_info.instances.append(instance)
return Service(network=network,
name=service_name,
subnetworks=subnetworks)
def get(self, name):
"""
Get a image named "name" and return some data about it.
"""
for image in self.list():
if image.name == name:
logger.debug("Got image: %s", image)
return image
return None
def destroy(self, image):
"""
Destroy a image given by "image".
"""
for gce_image in self.driver.list_images(self.driver.project):
logger.debug("Raw image destroy: %s", gce_image)
if gce_image.id == image.image_id:
logger.debug("Destroying image: %s", gce_image)
return self.driver.ex_delete_image(gce_image)
return None
def get(self, name):
"""
Get a network named "name" and return some data about it.
"""
try:
network = self.driver.ex_get_network(name)
except ResourceNotFoundError as not_found:
logger.debug("Caught exception trying to get network: %s", not_found)
return None
return canonicalize_network_info(network)
def has_access(self, source, destination, port):
"""
Return true if there's a path between the services.
"""
logger.debug('Looking for path from %s to %s on port %s', source,
destination, 80)
self._validate_args(source, destination)
paths = self.list()
logger.debug('Found paths %s', paths)
return self._has_access(paths, source, destination, port)
def destroy(self, network):
"""
Destroy a network given by "network".
"""
try:
network = self.driver.ex_get_network(network.name)
except ResourceNotFoundError as not_found:
logger.debug("Caught exception destroying network, ignoring: %s",
not_found)
return None
return self.driver.ex_destroy_network(network)
def list(self):
"""
List all images.
"""
images = []
for gce_image in self.driver.list_images(self.driver.project):
logger.debug("Raw image list: %s", gce_image)
images.append(
Image(name=gce_image.name,
image_id=gce_image.id,
created_at=str(gce_image.extra["creationTimestamp"])))
return images
def get_gce_driver(credentials):
"""
Uses the given credentials to get a GCE driver object from libcloud.
"""
# pylint:disable=global-statement
global DRIVER
if not DRIVER:
logger.debug("GCE driver not initialized, creating.")
compute_engine_driver = get_driver(Provider.GCE)
DRIVER = compute_engine_driver(user_id=credentials["user_id"],
key=credentials["key"],
project=credentials["project"])
return DRIVER
def remove(self, source, destination, port):
"""
Remove path between two services on a given port.
"""
logger.debug('Removing path from %s to %s on port %s', source,
destination, port)
firewall_name = "bu-%s-%s-%s" % (destination.network.name,
destination.name, port)
def remove_from_ranges(to_remove, address_ranges):
logger.debug("Removing %s from %s", to_remove, address_ranges)
resulting_ranges = []
if not address_ranges:
return None
for address_range in address_ranges:
remove_net = ipaddress.IPv4Network(to_remove)
address_range_network = ipaddress.IPv4Network(address_range)
if remove_net.overlaps(address_range_network):
if remove_net.prefixlen > address_range_network.prefixlen:
new_range_networks = address_range_network.address_exclude(
remove_net)
resulting_ranges.extend([
str(new_range_network)
for new_range_network in new_range_networks
])
else:
resulting_ranges.extend([str(address_range_network)])
logger.debug("New ranges: %s", resulting_ranges)
return resulting_ranges
try:
firewall = self.driver.ex_get_firewall(firewall_name)
if isinstance(source, CidrBlock):
firewall.source_ranges = remove_from_ranges(
source.cidr_block, firewall.source_ranges)
else:
source_tag = "%s-%s" % (source.network.name, source.name)
if firewall.source_tags:
firewall.source_tags = [
tag for tag in firewall.source_tags
if tag != source_tag
]
except ResourceNotFoundError:
logger.debug("Firewall %s doesn't exist", firewall_name)
return None
# We need this because the default is to add "0.0.0.0/0" if these aren't set, which is bad.
if not firewall.source_tags and not firewall.source_ranges:
return self.driver.ex_destroy_firewall(firewall)
return self.driver.ex_update_firewall(firewall)
def list(self):
"""
List all instance groups.
"""
logger.debug('Listing services')
subnetworks = self.subnetwork.list()
services = []
for network_name, subnet_info in subnetworks.items():
logger.debug("Subnets in network %s: %s", network_name,
subnet_info)
for subnetwork_name, _ in subnet_info.items():
# Things might have changed from the time we listed the services, so skip if we
# can't find them anymore.
network = self.network.get(network_name)
if not network:
logger.debug("Network %s not found! %s", network_name,
subnet_info)
continue
service = self.get(network, subnetwork_name)
if not service:
logger.debug("Service %s not found! %s", subnetwork_name,
subnet_info)
continue
services.append(service)
return services
def add(self, source, destination, port):
"""
Add path between two services on a given port.
"""
logger.debug('Adding path from %s to %s on port %s', source,
destination, port)
rules = [{"IPProtocol": "tcp", "ports": [int(port)]}]
src_tags, dest_tags, src_ranges, _ = self._extract_service_info(
source, destination)
firewall_name = "bu-%s-%s-%s" % (destination.network.name,
destination.name, port)
try:
firewall = self.driver.ex_get_firewall(firewall_name)
if isinstance(source, CidrBlock):
if not firewall.source_ranges:
firewall.source_ranges = []
firewall.source_ranges.append(str(source.cidr_block))
logger.debug(firewall.source_ranges)
if isinstance(source, Service):
if not firewall.source_tags:
firewall.source_tags = []
source_tag = "%s-%s" % (source.network.name, source.name)
firewall.source_tags.append(source_tag)
logger.debug(firewall.source_tags)
firewall = self.driver.ex_update_firewall(firewall)
except ResourceNotFoundError:
logger.debug("Firewall %s not found, creating.", firewall_name)
firewall = self.driver.ex_create_firewall(
firewall_name,
allowed=rules,
network=destination.network.name,
source_ranges=src_ranges,
source_tags=src_tags,
target_tags=dest_tags)
return Path(destination.network, source, destination, "tcp", port)
def remove_from_ranges(to_remove, address_ranges):
logger.debug("Removing %s from %s", to_remove, address_ranges)
resulting_ranges = []
if not address_ranges:
return None
for address_range in address_ranges:
remove_net = ipaddress.IPv4Network(to_remove)
address_range_network = ipaddress.IPv4Network(address_range)
if remove_net.overlaps(address_range_network):
if remove_net.prefixlen > address_range_network.prefixlen:
new_range_networks = address_range_network.address_exclude(
remove_net)
resulting_ranges.extend([
str(new_range_network)
for new_range_network in new_range_networks
])
else:
resulting_ranges.extend([str(address_range_network)])
logger.debug("New ranges: %s", resulting_ranges)
return resulting_ranges
def destroy(self, service):
"""
Destroy a service described by "service".
"""
logger.debug('Destroying service: %s', service)
destroy_results = []
for node in self.driver.list_nodes():
metadata = node.extra.get("metadata", {}).get("items", [])
node_network_name = None
node_subnetwork_name = None
for item in metadata:
logger.debug("Found metadata item %s for node %s", item, node)
if item["key"] == "network":
node_network_name = item["value"]
if item["key"] == "subnetwork":
node_subnetwork_name = item["value"]
if (service.network.name == node_network_name
and service.name == node_subnetwork_name):
logger.info('Destroying instance: %s', node.name)
destroy_results.append(self.driver.destroy_node(node))
subnetwork_destroy = self.subnetwork.destroy(service.network.name,
service.name)
self.firewalls.delete_firewall(service.network.name, service.name)
return {"Subnetwork": subnetwork_destroy, "Instances": destroy_results}
def create(self, network, service_name, blueprint, template_vars, count):
"""
Create a service in "network" named "service_name" with blueprint file at "blueprint".
"""
logger.debug(
'Creating service %s, %s with blueprint %s and '
'template_vars %s', network.name, service_name, blueprint,
template_vars)
self.subnetwork.create(network.name, service_name, blueprint=blueprint)
instances_blueprint = ServiceBlueprint.from_file(blueprint)
az_count = instances_blueprint.availability_zone_count()
availability_zones = list(
itertools.islice(self._get_availability_zones(), az_count))
if len(availability_zones) < az_count:
raise DisallowedOperationException(
"Do not have %s availability zones: %s" %
(az_count, availability_zones))
instance_count = az_count
if count:
instance_count = count
def get_image(image_specifier):
images = [
image for image in self.driver.list_images()
if re.match(image_specifier, image.name)
]
if not images:
raise DisallowedOperationException(
"Could not find image named %s" % image_specifier)
if len(images) > 1:
raise DisallowedOperationException(
"Found multiple images for specifier %s: %s" %
(image_specifier, images))
return images[0]
image = get_image(instances_blueprint.image())
instance_type = get_fitting_instance(self, instances_blueprint)
for availability_zone, instance_num in zip(
itertools.cycle(availability_zones), range(0, instance_count)):
full_subnetwork_name = "%s-%s" % (network.name, service_name)
instance_name = "%s-%s" % (full_subnetwork_name, instance_num)
metadata = [{
"key":
"startup-script",
"value":
instances_blueprint.runtime_scripts(template_vars)
}, {
"key": "network",
"value": network.name
}, {
"key": "subnetwork",
"value": service_name
}]
logger.info('Creating instance %s in zone %s', instance_name,
availability_zone.name)
self.driver.create_node(instance_name,
instance_type,
image,
location=availability_zone,
ex_network=network.name,
ex_subnetwork=full_subnetwork_name,
external_ip="ephemeral",
ex_metadata=metadata,
ex_tags=[full_subnetwork_name])
return self.get(network, service_name)