def do_polish(polisher): engine = PlumberyEngine(myInformation) engine.set_shared_secret('fake_secret') engine.set_user_name('fake_name') engine.set_user_password('fake_password') polisher.go(engine) facility = engine.list_facility('NA9')[0] DimensionDataNodeDriver.connectionCls.conn_classes = ( None, DimensionDataMockHttp) DimensionDataMockHttp.type = None facility.region = DimensionDataNodeDriver(*DIMENSIONDATA_PARAMS) polisher.move_to(facility) blueprint = facility.get_blueprint('test') infrastructure = PlumberyInfrastructure(facility) container = infrastructure.get_container(blueprint) polisher.shine_container(container) nodes = PlumberyNodes(facility) node = nodes.get_node('stackstorm') polisher.shine_node(node=node, settings=fakeNodeSettings, container=container) node = nodes.get_node('node1') polisher.shine_node(node=node, settings=fakeNodeSettings, container=container) polisher.move_to(FakeFacility()) polisher.shine_container(FakeContainer()) polisher.shine_node(node=FakeNode(), settings=fakeNodeSettings, container=FakeContainer()) polisher.reap()
def do_polish(polisher): engine = PlumberyEngine() engine.set_shared_secret('fake_secret') engine.set_user_name('fake_name') engine.set_user_password('fake_password') engine.from_text(myInformation) polisher.go(engine) facility = engine.list_facility('NA9')[0] DimensionDataNodeDriver.connectionCls.conn_classes = ( None, DimensionDataMockHttp) DimensionDataMockHttp.type = None facility.region = DimensionDataNodeDriver(*DIMENSIONDATA_PARAMS) polisher.move_to(facility) blueprint = facility.get_blueprint('test') infrastructure = PlumberyInfrastructure(facility) container = infrastructure.get_container(blueprint) polisher.shine_container(container) nodes = PlumberyNodes(facility) node = nodes.get_node('stackstorm') polisher.shine_node( node=node, settings=fakeNodeSettings, container=container) node = nodes.get_node('node1') polisher.shine_node( node=node, settings=fakeNodeSettings, container=container) polisher.move_to(FakeFacility()) polisher.shine_container(FakeContainer()) polisher.shine_node( node=FakeNode(), settings=fakeNodeSettings, container=FakeContainer()) polisher.reap()
def lookup(self, token): """ Retrieves the value attached to a token :param token: the token, e.g., 'node.ipv6' :type token: ``str`` :return: the value attached to this token, or `None` """ if token in self.cache: return str(self.cache[token]) value = None if self.context is not None: value = self.context.lookup(token) if value is not None: return value if self.container is None: return None tokens = token.split('.') if len(tokens) < 2: tokens.append('private') nodes = PlumberyNodes(self.container.facility) node = nodes.get_node(tokens[0]) if node is None: return None if self.context is not None: self.context.remember(tokens[0], node.private_ips[0]) self.context.remember(tokens[0] + '.private', node.private_ips[0]) self.context.remember(tokens[0] + '.ipv6', node.extra['ipv6']) if len(node.public_ips) > 0: self.context.remember(tokens[0] + '.public', node.public_ips[0]) if tokens[1] == 'private': return node.private_ips[0] if tokens[1] == 'ipv6': return node.extra['ipv6'] if tokens[1] == 'public': if len(node.public_ips) > 0: return node.public_ips[0] else: return '' return None
def lookup(self, token): """ Retrieves the value attached to a token :param token: the token, e.g., 'node.ipv6' :type token: ``str`` :return: the value attached to this token, or `None` """ if token in self.cache: return str(self.cache[token]) value = None if self.context is not None: value = self.context.lookup(token) if value is not None: return value if self.container is None: return None tokens = token.split(".") if len(tokens) < 2: tokens.append("private") nodes = PlumberyNodes(self.container.facility) node = nodes.get_node(tokens[0]) if node is None: return None if self.context is not None: self.context.remember(tokens[0], node.private_ips[0]) self.context.remember(tokens[0] + ".private", node.private_ips[0]) self.context.remember(tokens[0] + ".ipv6", node.extra["ipv6"]) if len(node.public_ips) > 0: self.context.remember(tokens[0] + ".public", node.public_ips[0]) if tokens[1] == "private": return node.private_ips[0] if tokens[1] == "ipv6": return node.extra["ipv6"] if tokens[1] == "public": if len(node.public_ips) > 0: return node.public_ips[0] else: return "" return None
def shine_container(self, container): """ Rubs a container until it shines :param container: the container to be polished :type container: :class:`plumbery.PlumberyInfrastructure` This is where the hard work is done. You have to override this function in your own polisher. Note that you can compare the reality versus the theoritical settings if you want. """ logging.info("Spitting at blueprint '{}'".format( container.blueprint['target'])) if container.network is None: logging.info("- aborted - no network here") return nodes = PlumberyNodes(self.facility) names = nodes.list_nodes(container.blueprint) logging.info("Waiting for nodes of '{}' to be deployed".format( container.blueprint['target'])) for name in names: while True: node = nodes.get_node(name) if node is None: logging.info("- aborted - missing node '{}'".format(name)) return if node.extra['status'].action is None: break if (node is not None and node.extra['status'].failure_reason is not None): logging.info("- aborted - failed deployment " "of node '{}'".format(name)) return time.sleep(20) logging.info("- done") container._build_firewall_rules() container._reserve_ipv4() container._build_balancer()
class TestPlumberyNodes(unittest.TestCase): def setUp(self): self.nodes = PlumberyNodes(FakeFacility()) def tearDown(self): self.nodes = None def test_build_blueprint(self): domain = FakeDomain() self.nodes.build_blueprint(fakeBlueprint, domain) def test_destroy_blueprint(self): self.nodes.destroy_blueprint(fakeBlueprint) def test_get_node(self): self.nodes.get_node("stackstorm") def test_start_nodes(self): self.nodes.start_blueprint("fake") def test_stop_nodes(self): self.nodes.stop_blueprint("fake")
class TestPlumberyNodes(unittest.TestCase): def setUp(self): self.nodes = PlumberyNodes(FakeFacility()) def tearDown(self): self.nodes = None def test_build_blueprint(self): domain = FakeDomain() self.nodes.build_blueprint(fakeBlueprint, domain) # def test_destroy_blueprint(self): # self.nodes.destroy_blueprint(fakeBlueprint) def test_get_node(self): self.nodes.get_node('stackstorm') def test_start_nodes(self): self.nodes.start_blueprint('fake') def test_stop_nodes(self): self.nodes.stop_blueprint('fake')
def shine_container(self, container): """ Rubs a container until it shines :param container: the container to be polished :type container: :class:`plumbery.PlumberyInfrastructure` This is where the hard work is done. You have to override this function in your own polisher. Note that you can compare the reality versus the theoritical settings if you want. """ if container.network is None: return nodes = PlumberyNodes(self.facility) names = nodes.list_nodes(container.blueprint) logging.info("Waiting for nodes to be deployed") for name in names: while True: node = nodes.get_node(name) if node is None: logging.info("- aborted - missing node '{}'".format(name)) return if node.extra['status'].action is None: break if node is not None \ and node.extra['status'].failure_reason is not None: logging.info( "- aborted - failed deployment of node '{}'".format( name)) return time.sleep(20) logging.info("- done") container._build_firewall_rules() container._reserve_ipv4() container._build_balancer()
class ConfigurePolisher(PlumberyPolisher): """ Finalizes the setup of fittings This polisher looks at each node in sequence, and adjust settings according to fittings plan. This is covering various features that can not be set during the creation of nodes, such as: - number of CPU - quantity of RAM - monitoring """ def move_to(self, facility): """ Moves to another API endpoint :param facility: access to local parameters and functions :type facility: :class:`plumbery.PlumberyFacility` """ self.facility = facility self.region = facility.region self.nodes = PlumberyNodes(facility) def shine_container(self, container): """ prepares a container until it shines :param container: the container to be polished :type container: :class:`plumbery.PlumberyInfrastructure` This is where the hard work is done. You have to override this function in your own polisher. Note that you can compare the reality versus the theoritical settings if you want. """ logging.info("Configuring blueprint '{}'".format( container.blueprint['target'])) if container.network is None: logging.info("- aborted - no network here") return logging.info("- waiting for nodes to be deployed") names = self.nodes.list_nodes(container.blueprint) for name in names: while True: node = self.nodes.get_node(name) if node is None: logging.info("- aborted - missing node '{}'".format(name)) return if node.extra['status'].action is None: break if (node is not None and node.extra['status'].failure_reason is not None): logging.info("- aborted - failed deployment " "of node '{}'".format(name)) return time.sleep(20) logging.info("- nodes have been deployed") container._build_firewall_rules() container._build_balancer() def set_node_compute(self, node, cpu, memory): """ Sets compute capability :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param cpu: the cpu specification :type cpu: ``DimensionDataServerCpuSpecification`` :param memory: the memory size, expressed in Giga bytes :type memory: ``int`` """ changed = False if cpu is not None and 'cpu' in node.extra: if int(cpu.cpu_count) != int(node.extra['cpu'].cpu_count): logging.info("- changing to {} cpu".format(cpu.cpu_count)) changed = True if int(cpu.cores_per_socket) != int( node.extra['cpu'].cores_per_socket): logging.info("- changing to {} core(s) per socket".format( cpu.cores_per_socket)) changed = True if cpu.performance != node.extra['cpu'].performance: logging.info("- changing to '{}' cpu performance".format( cpu.performance.lower())) changed = True if memory is not None and 'memoryMb' in node.extra: if memory != int(node.extra['memoryMb'] / 1024): logging.info("- changing to {} GB memory".format(memory)) changed = True if not changed: logging.debug("- no change in compute") return if self.engine.safeMode: logging.info("- skipped - safe mode") return while True: try: self.region.ex_reconfigure_node( node=node, memory_gb=memory, cpu_count=cpu.cpu_count, cores_per_socket=cpu.cores_per_socket, cpu_performance=cpu.performance) logging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue logging.info("- unable to reconfigure node") logging.error(str(feedback)) break def change_node_disk_size(self, node, id, size): """ Changes an existing virtual disk :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param id: the disk unique identifier, as reported by the API :type id: ``str`` :param size: the disk size, expressed in Giga bytes :type size: ``int`` """ if self.engine.safeMode: logging.info("- skipped - safe mode") return while True: try: self.region.ex_change_storage_size(node=node, disk_id=id, size=size) logging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue logging.info( "- unable to change disk size to {}GB".format(size)) logging.error(str(feedback)) break def change_node_disk_speed(self, node, id, speed): """ Changes an existing virtual disk :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param id: the disk unique identifier, as reported by the API :type id: ``str`` :param speed: storage type, either 'standard', 'highperformance' or 'economy' :type speed: ``str`` """ if self.engine.safeMode: logging.info("- skipped - safe mode") return while True: try: self.region.ex_change_storage_speed(node=node, disk_id=id, speed=speed) logging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue logging.info("- unable to change disk to '{}'".format(speed)) logging.error(str(feedback)) break def set_node_disk(self, node, id, size, speed='standard'): """ Sets a virtual disk :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param id: the disk id, starting at 0 and growing :type id: ``int`` :param size: the disk size, expressed in Giga bytes :type size: ``int`` :param speed: storage type, either 'standard', 'highperformance' or 'economy' :type speed: ``str`` """ if size < 1: logging.info("- minimum disk size is 1 GB") return if size > 1000: logging.info("- disk size cannot exceed 1000 GB") return if speed not in ['standard', 'highperformance', 'economy']: logging.info("- disk speed should be either 'standard' " "or 'highperformance' or 'economy'") return if 'disks' in node.extra: for disk in node.extra['disks']: if disk['scsiId'] == id: changed = False if disk['size'] > size: logging.info("- disk shrinking could break the node") logging.info( "- skipped - disk {} will not be reduced".format( id)) if disk['size'] < size: logging.info("- expanding disk {} to {} GB".format( id, size)) self.change_node_disk_size(node, disk['id'], size) changed = True if disk['speed'].lower() != speed.lower(): logging.info("- changing disk {} to '{}'".format( id, speed)) self.change_node_disk_speed(node, disk['id'], speed) changed = True if not changed: logging.debug("- no change in disk {}".format(id)) return logging.info("- adding {} GB '{}' disk".format(size, speed)) if self.engine.safeMode: logging.info("- skipped - safe mode") return while True: try: self.region.ex_add_storage_to_node(node=node, amount=size, speed=speed.upper()) logging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue logging.info("- unable to add disk {} GB '{}'".format( size, speed)) logging.error(str(feedback)) break def shine_node(self, node, settings, container): """ Finalizes setup of one node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param settings: the fittings plan for this node :type settings: ``dict`` :param container: the container of this node :type container: :class:`plumbery.PlumberyInfrastructure` """ logging.info("Configuring node '{}'".format(settings['name'])) if node is None: logging.info("- not found") return cpu = None if 'cpu' in settings: tokens = str(settings['cpu']).split(' ') if len(tokens) < 2: tokens.append('1') if len(tokens) < 3: tokens.append('standard') if (int(tokens[0]) < 1 or int(tokens[0]) > 32): logging.info("- cpu should be between 1 and 32") elif (int(tokens[1]) < 1 or int(tokens[1]) > 2): logging.info("- core per cpu should be either 1 or 2") elif tokens[2].upper() not in ('STANDARD', 'HIGHPERFORMANCE'): logging.info("- cpu speed should be either 'standard'" " or 'highspeed'") else: logging.debug("- setting compute {}".format(' '.join(tokens))) cpu = DimensionDataServerCpuSpecification( cpu_count=tokens[0], cores_per_socket=tokens[1], performance=tokens[2].upper()) memory = None if 'memory' in settings: memory = int(settings['memory']) if memory < 1 or memory > 256: logging.info("- memory should be between 1 and 256") memory = None else: logging.debug("- setting {} GB of memory".format(memory)) self.set_node_compute(node, cpu, memory) if 'disks' in settings: for item in settings['disks']: logging.debug("- setting disk {}".format(item)) attributes = item.split() if len(attributes) < 2: logging.info("- malformed disk attributes;" " provide disk id and size in GB, e.g., 1 50;" " add disk type if needed, e.g., economy") elif len(attributes) < 3: id = int(attributes[0]) size = int(attributes[1]) speed = 'standard' else: id = int(attributes[0]) size = int(attributes[1]) speed = attributes[2] self.set_node_disk(node, id, size, speed) if 'monitoring' in settings: self.nodes._start_monitoring(node, settings['monitoring']) if 'backup' in settings: self.nodes._configure_backup(node, settings['backup']) if 'glue' in settings: container._attach_node(node, settings['glue']) container._add_to_pool(node)
class ConfigurePolisher(PlumberyPolisher): """ Configures various elements in fittings plan This polisher looks at each node in sequence, and adjust settings according to fittings plan. This is covering various features that can not be set during the creation of nodes, such as: - number of CPU - quantity of RAM - monitoring - network interfaces """ configuration_props = (MonitoringConfiguration, DisksConfiguration, BackupConfiguration, WindowsConfiguration) def move_to(self, facility): """ Moves to another API endpoint :param facility: access to local parameters and functions :type facility: :class:`plumbery.PlumberyFacility` """ self.facility = facility self.region = facility.region self.nodes = PlumberyNodes(facility) def shine_container(self, container): """ Configures a container :param container: the container to be polished :type container: :class:`plumbery.PlumberyInfrastructure` """ plogging.info("Configuring blueprint '{}'".format( container.blueprint['target'])) if container.network is None: plogging.error("- aborted - no network here") return self.container = container plogging.info("- waiting for nodes to be deployed") names = self.nodes.list_nodes(container.blueprint) for name in sorted(names): while True: node = self.nodes.get_node(name) if node is None: plogging.error( "- aborted - missing node '{}'".format(name)) return if node.extra['status'].action is None: plogging.debug("- {} is ready".format(node.name)) break if (node is not None and node.extra['status'].failure_reason is not None): plogging.error("- aborted - failed deployment " "of node '{}'".format(name)) return time.sleep(20) plogging.info("- nodes have been deployed") container._build_firewall_rules() container._build_balancer() def set_node_compute(self, node, cpu, memory): """ Sets compute capability :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param cpu: the cpu specification :type cpu: ``DimensionDataServerCpuSpecification`` :param memory: the memory size, expressed in Giga bytes :type memory: ``int`` """ changed = False if cpu is not None and 'cpu' in node.extra: if int(cpu.cpu_count) != int(node.extra['cpu'].cpu_count): plogging.info("- changing to {} cpu".format(cpu.cpu_count)) changed = True if (int(cpu.cores_per_socket) != int( node.extra['cpu'].cores_per_socket)): plogging.info("- changing to {} core(s) per socket".format( cpu.cores_per_socket)) changed = True if cpu.performance != node.extra['cpu'].performance: plogging.info("- changing to '{}' cpu performance".format( cpu.performance.lower())) changed = True if memory is not None and 'memoryMb' in node.extra: if memory != int(node.extra['memoryMb'] / 1024): plogging.info("- changing to {} GB memory".format(memory)) changed = True if not changed: plogging.debug("- no change in compute") return if self.engine.safeMode: plogging.info("- skipped - safe mode") return while True: try: self.region.ex_reconfigure_node( node=node, memory_gb=memory, cpu_count=cpu.cpu_count, cores_per_socket=cpu.cores_per_socket, cpu_performance=cpu.performance) plogging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue plogging.info("- unable to reconfigure node") plogging.error(str(feedback)) break def attach_node(self, node, networks): """ Glues a node to multiple networks :param node: the target node :type node: :class:`libcloud.compute.base.Node` :param networks: a list of networks to connect, and ``internet`` :type networks: list of ``str`` This function adds network interfaces to a node, or adds address translation to the public Internet. Example in the fittings plan:: - web: domain: ipv4: 6 ethernet: name: gigafox.data nodes: - web[10..12]: glue: - gigafox.control - internet 80 443 In this example, another network interface is added to each node for connection to the Ethernet network ``gigafox.control``. Also, public IPv4 addresses are mapped on private addresses, so that each node web10, web11 and web12 is reachable from the internet. Public IPv4 addresses are taken from pool declared at the domain level, with the attribute ``ipv4``. In the example above, 6 addresses are assigned to the network domain, of which 3 are given to web nodes. If one or multiple numbers are mentioned after the keyword `internet`, they are used to configure the firewall appropriately. """ hasChanged = False if node is None: return hasChanged for line in networks: tokens = line.strip(' ').split(' ') token = tokens.pop(0) if token.lower() == 'internet': self.attach_node_to_internet(node, tokens) continue if token == self.container.blueprint['ethernet']['name']: continue if token.lower() == 'primary': continue plogging.info("Glueing node '{}' to network '{}'".format( node.name, token)) vlan = self.container.get_ethernet(token.split('::')) if vlan is None: plogging.info("- network '{}' is unknown".format(token)) continue kwargs = {} if len(tokens) > 0: numbers = tokens.pop(0).strip('.').split('.') subnet = vlan.private_ipv4_range_address.split('.') while len(numbers) < 4: numbers.insert(0, subnet[3 - len(numbers)]) private_ipv4 = '.'.join(numbers) plogging.debug("- using address '{}'".format(private_ipv4)) kwargs['private_ipv4'] = private_ipv4 if self.engine.safeMode: plogging.info("- skipped - safe mode") continue if 'private_ipv4' not in kwargs: kwargs['vlan'] = vlan while True: try: self.region.ex_attach_node_to_vlan(node, **kwargs) plogging.info("- in progress") hasChanged = True except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue elif 'RESOURCE_LOCKED' in str(feedback): plogging.info("- not now - locked") elif 'INVALID_INPUT_DATA' in str(feedback): plogging.info("- already there") else: plogging.info("- unable to glue node") plogging.error(str(feedback)) break return hasChanged def attach_node_to_internet(self, node, ports=[]): """ Adds address translation for one node :param node: node that has to be reachable from the internet :type node: :class:`libcloud.common.Node` :param ports: the ports that have to be opened :type ports: a list of ``str`` """ plogging.info("Making node '{}' reachable from the internet".format( node.name)) domain = self.container.get_network_domain( self.container.blueprint['domain']['name']) internal_ip = node.private_ips[0] external_ip = None for rule in self.region.ex_list_nat_rules(domain): if rule.internal_ip == internal_ip: external_ip = rule.external_ip plogging.info( "- node is reachable at '{}'".format(external_ip)) if self.engine.safeMode: plogging.info("- skipped - safe mode") return if external_ip is None: external_ip = self.container._get_ipv4() if external_ip is None: plogging.info( "- no more ipv4 address available -- assign more") return while True: try: self.region.ex_create_nat_rule(domain, internal_ip, external_ip) plogging.info( "- node is reachable at '{}'".format(external_ip)) except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue elif 'RESOURCE_LOCKED' in str(feedback): plogging.info("- not now - locked") return else: plogging.info("- unable to add address translation") plogging.error(str(feedback)) break candidates = self.container._list_candidate_firewall_rules(node, ports) for rule in self.container._list_firewall_rules(): if rule.name in candidates.keys(): plogging.info("Creating firewall rule '{}'".format(rule.name)) plogging.info("- already there") candidates = { k: candidates[k] for k in candidates if k != rule.name } for name, rule in candidates.items(): plogging.info("Creating firewall rule '{}'".format(name)) if self.engine.safeMode: plogging.info("- skipped - safe mode") else: try: self.container._ex_create_firewall_rule( network_domain=domain, rule=rule, position='LAST') plogging.info("- in progress") except Exception as feedback: if 'NAME_NOT_UNIQUE' in str(feedback): plogging.info("- already there") else: plogging.info("- unable to create firewall rule") plogging.error(str(feedback)) def shine_node(self, node, settings, container): """ Finalizes setup of one node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param settings: the fittings plan for this node :type settings: ``dict`` :param container: the container of this node :type container: :class:`plumbery.PlumberyInfrastructure` """ plogging.info("Configuring node '{}'".format(settings['name'])) if node is None: plogging.info("- not found") return try: cpu_prop = CpuConfiguration() cpu_prop.validate(settings) cpu = cpu_prop.configure(node, settings) ram_prop = MemoryConfiguration() ram_prop.validate(settings) memory = ram_prop.configure(node, settings) if memory is not False and cpu is not False: self.set_node_compute(node, cpu, memory) except ConfigurationError as ce: if self.engine.safeMode: plogging.warn(ce.message) else: raise ce for prop_cls in self.configuration_props: try: configuration_prop = prop_cls( engine=container.facility.plumbery, facility=self.facility) configuration_prop.validate(settings) configuration_prop.configure(node, settings) except ConfigurationError as ce: if self.engine.safeMode: plogging.warn(ce.message) else: raise ce container._add_to_pool(node) if 'glue' in settings: self.attach_node(node, settings['glue'])
class PreparePolisher(PlumberyPolisher): """ Bootstraps nodes via ssh This polisher looks at each node in sequence, and contact selected nodes via ssh to prepare them. The goal here is to accelerate post-creation tasks as much as possible. Bootstrapping steps can consist of multiple tasks: * push a SSH public key to allow for automated secured communications * ask for package update * install docker * install any pythons script * install Stackstorm * configure a Chef client * register a node to a monitoring dashboard * ... To activate this polisher you have to mention it in the fittings plan, like in the following example:: --- safeMode: False actions: - prepare: key: ~/.ssh/id_rsa.pub --- # Frankfurt in Europe locationId: EU6 regionId: dd-eu ... Plumbery will only prepare nodes that have been configured for it. The example below demonstrates how this can be done for multiple docker containers:: # some docker resources - docker: domain: *vdc1 ethernet: *containers nodes: - docker1: prepare: &docker - run prepare.update.sh - run prepare.docker.sh - docker2: prepare: *docker - docker3: prepare: *docker In the real life when you have to prepare any appliance, you need to be close to the stuff and to touch it. This is the same for virtual fittings. This polisher has the need to communicate directly with target nodes over the network. This connectivity can become quite complicated because of the potential mix of private and public networks, firewalls, etc. To stay safe plumbery enforces a simple beachheading model, where network connectivity with end nodes is a no brainer. This model is based on predefined network addresses for plumbery itself, as in the snippet below:: --- # Frankfurt in Europe locationId: EU6 regionId: dd-eu # network subnets are 10.1.x.y prepare: - beachhead: 10.1.3.4 Here nodes at EU6 will be prepared only if the machine that is executing plumbery has the adress 10.1.3.4. In other cases, plumbery will state that the location is out of reach. """ def upgrade_vmware_tools(self, node): """ Upgrade VMware tools on target node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` """ if self.engine.safeMode: return True while True: try: self.region.ex_update_vm_tools(node=node) plogging.info("- upgrading vmware tools") return True except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue if 'NO_CHANGE' in str(feedback): plogging.debug("- vmware tools is already up-to-date") return True plogging.warning("- unable to upgrade vmware tools") plogging.warning(str(feedback)) return False def _apply_prepares(self, node, steps): """ Does the actual job over SSH :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param steps: the various steps of the preparing :type steps: :class:`libcloud.compute.deployment.MultiStepDeployment` :return: ``True`` if everything went fine, ``False`` otherwise :rtype: ``bool`` """ if node is None or node.state != NodeState.RUNNING: plogging.warning("- skipped - node is not running") return False # select the address to use if len(node.public_ips) > 0: target_ip = node.public_ips[0] elif node.extra['ipv6']: target_ip = node.extra['ipv6'] else: target_ip = node.private_ips[0] # guess location of user key path = os.path.expanduser('~/.ssh/id_rsa') # use libcloud to communicate with remote nodes session = SSHClient(hostname=target_ip, port=22, username=self.user, password=self.secret, key_files=path, timeout=9) try: session.connect() except Exception as feedback: plogging.error("Error: unable to prepare '{}' at '{}'!".format( node.name, target_ip)) plogging.error(str(feedback)) plogging.error("- failed") return False while True: try: if self.engine.safeMode: plogging.info("- skipped - no ssh interaction in safe mode") else: node = steps.run(node, session) except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue plogging.error("Error: unable to prepare '{}' at '{}'!".format( node.name, target_ip)) plogging.error(str(feedback)) plogging.error("- failed") result = False else: result = True break try: session.close() except: pass return result def _get_prepares(self, node, settings, container): """ Defines the set of actions to be done on a node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param settings: the fittings plan for this node :type settings: ``dict`` :param container: the container of this node :type container: :class:`plumbery.PlumberyInfrastructure` :return: a list of actions to be performed, and related descriptions :rtype: a ``list`` of `{ 'description': ..., 'genius': ... }`` """ if not isinstance(settings, dict): return [] environment = PlumberyNodeContext(node=node, container=container, context=self.facility) prepares = [] if self.key is not None: prepares.append({ 'description': 'deploy SSH public key', 'genius': SSHKeyDeployment(self.key)}) if ('prepare' in settings and isinstance(settings['prepare'], list) and len(settings['prepare']) > 0): plogging.info('- using prepare commands') for script in settings['prepare']: tokens = script.split(' ') if len(tokens) == 1: tokens.insert(0, 'run') if tokens[0] in ['run', 'run_raw']: # send and run a script script = tokens[1] if len(tokens) > 2: args = tokens[2:] else: args = [] plogging.debug("- {} {} {}".format( tokens[0], script, ' '.join(args))) try: with open(script) as stream: text = stream.read() if(tokens[0] == 'run' and PlumberyText.could_expand(text)): plogging.debug("- expanding script '{}'" .format(script)) text = PlumberyText.expand_string( text, environment) if len(text) > 0: plogging.info("- running '{}'" .format(script)) prepares.append({ 'description': ' '.join(tokens), 'genius': ScriptDeployment( script=text, args=args, name=script)}) else: plogging.error("- script '{}' is empty" .format(script)) except IOError: plogging.error("- unable to read script '{}'" .format(script)) elif tokens[0] in ['put', 'put_raw']: # send a file file = tokens[1] if len(tokens) > 2: destination = tokens[2] else: destination = './'+file plogging.debug("- {} {} {}".format( tokens[0], file, destination)) try: with open(file) as stream: content = stream.read() if(tokens[0] == 'put' and PlumberyText.could_expand(content)): plogging.debug("- expanding file '{}'" .format(file)) content = PlumberyText.expand_string( content, environment) plogging.info("- putting file '{}'" .format(file)) prepares.append({ 'description': ' '.join(tokens), 'genius': FileContentDeployment( content=content, target=destination)}) except IOError: plogging.error("- unable to read file '{}'" .format(file)) else: # echo a sensible message eventually if tokens[0] == 'echo': tokens.pop(0) message = ' '.join(tokens) message = PlumberyText.expand_string( message, environment) plogging.info("- {}".format(message)) if ('cloud-config' in settings and isinstance(settings['cloud-config'], dict) and len(settings['cloud-config']) > 0): plogging.info('- using cloud-config') # mandatory, else cloud-init will not consider user-data plogging.debug('- preparing meta-data') meta_data = 'instance_id: dummy\n' destination = '/var/lib/cloud/seed/nocloud-net/meta-data' prepares.append({ 'description': 'put meta-data', 'genius': FileContentDeployment( content=meta_data, target=destination)}) plogging.debug('- preparing user-data') expanded = PlumberyText.expand_string( settings['cloud-config'], environment) user_data = '#cloud-config\n'+expanded plogging.debug(user_data) destination = '/var/lib/cloud/seed/nocloud-net/user-data' prepares.append({ 'description': 'put user-data', 'genius': FileContentDeployment( content=user_data, target=destination)}) plogging.debug('- preparing remote install of cloud-init') script = 'prepare.cloud-init.sh' try: path = os.path.dirname(__file__)+'/'+script with open(path) as stream: text = stream.read() if text: prepares.append({ 'description': 'run '+script, 'genius': ScriptDeployment( script=text, name=script)}) except IOError: raise PlumberyException("Error: cannot read '{}'" .format(script)) plogging.debug('- preparing reboot to trigger cloud-init') prepares.append({ 'description': 'reboot node', 'genius': RebootDeployment( container=container)}) return prepares def go(self, engine): """ Starts the prepare process :param engine: access to global parameters and functions :type engine: :class:`plumbery.PlumberyEngine` """ super(PreparePolisher, self).go(engine) self.report = [] self.user = engine.get_shared_user() self.secret = engine.get_shared_secret() self.key = None if 'key' in self.settings: try: path = os.path.expanduser(self.settings['key']) with open(path) as stream: self.key = stream.read() stream.close() except IOError: pass def move_to(self, facility): """ Checks if we can beachhead at this facility :param facility: access to local parameters and functions :type facility: :class:`plumbery.PlumberyFacility` This function lists all addresses of the computer that is running plumbery. If there is at least one routable IPv6 address, then it assumes that communication with nodes is possible. If no suitable IPv6 address can be found, then plumbery falls back to IPv4. Beachheading is granted only if the address of the computer running plumbery matches the fitting parameter ``beachhead``. """ self.facility = facility self.region = facility.region self.nodes = PlumberyNodes(facility) self.beachheading = False try: self.addresses = [] for interface in netifaces.interfaces(): addresses = netifaces.ifaddresses(interface) if netifaces.AF_INET in addresses.keys(): for address in addresses[netifaces.AF_INET]: # strip local loop if address['addr'].startswith('127.0.0.1'): continue self.addresses.append(address['addr']) if netifaces.AF_INET6 in addresses.keys(): for address in addresses[netifaces.AF_INET6]: # strip local loop if address['addr'].startswith('::1'): continue # strip local link addresses if address['addr'].startswith('fe80::'): continue # we have a routable ipv6, so let's go self.beachheading = True except Exception as feedback: plogging.error(str(feedback)) for item in self.facility.get_setting('prepare', []): if not isinstance(item, dict): continue if 'beachhead' not in item.keys(): continue if item['beachhead'] in self.addresses: self.beachheading = True break if self.beachheading: plogging.debug("- beachheading at '{}'".format( self.facility.get_setting('locationId'))) else: plogging.debug("- not beachheading at '{}'".format( self.facility.get_setting('locationId'))) def shine_node(self, node, settings, container): """ prepares a node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param settings: the fittings plan for this node :type settings: ``dict`` :param container: the container of this node :type container: :class:`plumbery.PlumberyInfrastructure` """ plogging.info("Preparing node '{}'".format(settings['name'])) if node is None: plogging.error("- not found") return timeout = 300 tick = 6 while node.extra['status'].action == 'START_SERVER': time.sleep(tick) node = self.nodes.get_node(node.name) timeout -= tick if timeout < 0: break if node.state != NodeState.RUNNING: plogging.error("- skipped - node is not running") return self.upgrade_vmware_tools(node) prepares = self._get_prepares(node, settings, container) if len(prepares) < 1: plogging.info('- nothing to do') self.report.append({node.name: { 'status': 'skipped - nothing to do' }}) return if len(node.public_ips) > 0: plogging.info("- node is reachable at '{}'".format( node.public_ips[0])) elif not self.beachheading: plogging.error('- node is unreachable') self.report.append({node.name: { 'status': 'unreachable' }}) return descriptions = [] steps = [] for item in prepares: descriptions.append(item['description']) steps.append(item['genius']) if self._apply_prepares(node, MultiStepDeployment(steps)): plogging.info('- rebooting') self.report.append({node.name: { 'status': 'completed', 'prepares': descriptions }}) else: self.report.append({node.name: { 'status': 'failed', 'prepares': descriptions }}) def reap(self): """ Reports on preparing """ if 'output' not in self.settings: return fileName = self.settings['output'] plogging.info("Reporting on preparations in '{}'".format(fileName)) with open(fileName, 'w') as stream: stream.write(yaml.dump(self.report, default_flow_style=False)) stream.close()
class ConfigurePolisher(PlumberyPolisher): """ Configures various elements in fittings plan This polisher looks at each node in sequence, and adjust settings according to fittings plan. This is covering various features that can not be set during the creation of nodes, such as: - number of CPU - quantity of RAM - monitoring - network interfaces """ configuration_props = (MonitoringConfiguration, DisksConfiguration, BackupConfiguration, WindowsConfiguration) def move_to(self, facility): """ Moves to another API endpoint :param facility: access to local parameters and functions :type facility: :class:`plumbery.PlumberyFacility` """ self.facility = facility self.region = facility.region self.nodes = PlumberyNodes(facility) def shine_container(self, container): """ Configures a container :param container: the container to be polished :type container: :class:`plumbery.PlumberyInfrastructure` """ plogging.info("Configuring blueprint '{}'".format( container.blueprint['target'])) if container.network is None: plogging.error("- aborted - no network here") return self.container = container plogging.info("- waiting for nodes to be deployed") names = self.nodes.list_nodes(container.blueprint) for name in sorted(names): while True: node = self.nodes.get_node(name) if node is None: plogging.error("- aborted - missing node '{}'".format(name)) return if node.extra['status'].action is None: plogging.debug("- {} is ready".format(node.name)) break if (node is not None and node.extra['status'].failure_reason is not None): plogging.error("- aborted - failed deployment " "of node '{}'".format(name)) return time.sleep(20) plogging.info("- nodes have been deployed") container._build_firewall_rules() container._build_balancer() def set_node_compute(self, node, cpu, memory): """ Sets compute capability :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param cpu: the cpu specification :type cpu: ``DimensionDataServerCpuSpecification`` :param memory: the memory size, expressed in Giga bytes :type memory: ``int`` """ changed = False if cpu is not None and 'cpu' in node.extra: if int(cpu.cpu_count) != int(node.extra['cpu'].cpu_count): plogging.info("- changing to {} cpu".format( cpu.cpu_count)) changed = True if (int(cpu.cores_per_socket) != int(node.extra['cpu'].cores_per_socket)): plogging.info("- changing to {} core(s) per socket".format( cpu.cores_per_socket)) changed = True if cpu.performance != node.extra['cpu'].performance: plogging.info("- changing to '{}' cpu performance".format( cpu.performance.lower())) changed = True if memory is not None and 'memoryMb' in node.extra: if memory != int(node.extra['memoryMb']/1024): plogging.info("- changing to {} GB memory".format( memory)) changed = True if not changed: plogging.debug("- no change in compute") return if self.engine.safeMode: plogging.info("- skipped - safe mode") return while True: try: self.region.ex_reconfigure_node( node=node, memory_gb=memory, cpu_count=cpu.cpu_count, cores_per_socket=cpu.cores_per_socket, cpu_performance=cpu.performance) plogging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue plogging.info("- unable to reconfigure node") plogging.error(str(feedback)) break def attach_node(self, node, networks): """ Glues a node to multiple networks :param node: the target node :type node: :class:`libcloud.compute.base.Node` :param networks: a list of networks to connect, and ``internet`` :type networks: list of ``str`` This function adds network interfaces to a node, or adds address translation to the public Internet. Example in the fittings plan:: - web: domain: ipv4: 6 ethernet: name: gigafox.data nodes: - web[10..12]: glue: - gigafox.control - internet 80 443 In this example, another network interface is added to each node for connection to the Ethernet network ``gigafox.control``. Also, public IPv4 addresses are mapped on private addresses, so that each node web10, web11 and web12 is reachable from the internet. Public IPv4 addresses are taken from pool declared at the domain level, with the attribute ``ipv4``. In the example above, 6 addresses are assigned to the network domain, of which 3 are given to web nodes. If one or multiple numbers are mentioned after the keyword `internet`, they are used to configure the firewall appropriately. """ hasChanged = False if node is None: return hasChanged for line in networks: tokens = line.strip(' ').split(' ') token = tokens.pop(0) if token.lower() == 'internet': self.attach_node_to_internet(node, tokens) continue if token == self.container.blueprint['ethernet']['name']: continue if token.lower() == 'primary': continue plogging.info("Glueing node '{}' to network '{}'" .format(node.name, token)) vlan = self.container.get_ethernet(token.split('::')) if vlan is None: plogging.info("- network '{}' is unknown".format(token)) continue kwargs = {} if len(tokens) > 0: numbers = tokens.pop(0).strip('.').split('.') subnet = vlan.private_ipv4_range_address.split('.') while len(numbers) < 4: numbers.insert(0, subnet[3-len(numbers)]) private_ipv4 = '.'.join(numbers) plogging.debug("- using address '{}'".format(private_ipv4)) kwargs['private_ipv4'] = private_ipv4 if self.engine.safeMode: plogging.info("- skipped - safe mode") continue if 'private_ipv4' not in kwargs: kwargs['vlan'] = vlan while True: try: self.region.ex_attach_node_to_vlan(node, **kwargs) plogging.info("- in progress") hasChanged = True except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue elif 'RESOURCE_LOCKED' in str(feedback): plogging.info("- not now - locked") elif 'INVALID_INPUT_DATA' in str(feedback): plogging.info("- already there") else: plogging.info("- unable to glue node") plogging.error(str(feedback)) break return hasChanged def attach_node_to_internet(self, node, ports=[]): """ Adds address translation for one node :param node: node that has to be reachable from the internet :type node: :class:`libcloud.common.Node` :param ports: the ports that have to be opened :type ports: a list of ``str`` """ plogging.info("Making node '{}' reachable from the internet" .format(node.name)) domain = self.container.get_network_domain( self.container.blueprint['domain']['name']) internal_ip = node.private_ips[0] external_ip = None for rule in self.region.ex_list_nat_rules(domain): if rule.internal_ip == internal_ip: external_ip = rule.external_ip plogging.info("- node is reachable at '{}'".format(external_ip)) if self.engine.safeMode: plogging.info("- skipped - safe mode") return if external_ip is None: external_ip = self.container._get_ipv4() if external_ip is None: plogging.info("- no more ipv4 address available -- assign more") return while True: try: self.region.ex_create_nat_rule( domain, internal_ip, external_ip) plogging.info("- node is reachable at '{}'".format( external_ip)) except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue elif 'RESOURCE_LOCKED' in str(feedback): plogging.info("- not now - locked") return else: plogging.info("- unable to add address translation") plogging.error(str(feedback)) break candidates = self.container._list_candidate_firewall_rules(node, ports) for rule in self.container._list_firewall_rules(): if rule.name in candidates.keys(): plogging.info("Creating firewall rule '{}'" .format(rule.name)) plogging.info("- already there") candidates = {k: candidates[k] for k in candidates if k != rule.name} for name, rule in candidates.items(): plogging.info("Creating firewall rule '{}'" .format(name)) if self.engine.safeMode: plogging.info("- skipped - safe mode") else: try: self.container._ex_create_firewall_rule( network_domain=domain, rule=rule, position='LAST') plogging.info("- in progress") except Exception as feedback: if 'NAME_NOT_UNIQUE' in str(feedback): plogging.info("- already there") else: plogging.info("- unable to create firewall rule") plogging.error(str(feedback)) def shine_node(self, node, settings, container): """ Finalizes setup of one node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param settings: the fittings plan for this node :type settings: ``dict`` :param container: the container of this node :type container: :class:`plumbery.PlumberyInfrastructure` """ plogging.info("Configuring node '{}'".format(settings['name'])) if node is None: plogging.info("- not found") return try: cpu_prop = CpuConfiguration() cpu_prop.validate(settings) cpu = cpu_prop.configure(node, settings) ram_prop = MemoryConfiguration() ram_prop.validate(settings) memory = ram_prop.configure(node, settings) if memory is not False and cpu is not False: self.set_node_compute(node, cpu, memory) except ConfigurationError as ce: if self.engine.safeMode: plogging.warn(ce.message) else: raise ce for prop_cls in self.configuration_props: try: configuration_prop = prop_cls(engine=container.facility.plumbery, facility=self.facility) configuration_prop.validate(settings) configuration_prop.configure(node, settings) except ConfigurationError as ce: if self.engine.safeMode: plogging.warn(ce.message) else: raise ce container._add_to_pool(node) if 'glue' in settings: self.attach_node(node, settings['glue'])
class PreparePolisher(PlumberyPolisher): """ Bootstraps nodes via ssh This polisher looks at each node in sequence, and contact selected nodes via ssh to prepare them. The goal here is to accelerate post-creation tasks as much as possible. Bootstrapping steps can consist of multiple tasks: * push a SSH public key to allow for automated secured communications * ask for package update * install docker * install any pythons script * install Stackstorm * configure a Chef client * register a node to a monitoring dashboard * ... To activate this polisher you have to mention it in the fittings plan, like in the following example:: --- safeMode: False actions: - prepare: key: ~/.ssh/myproject_rsa.pub --- # Frankfurt in Europe locationId: EU6 regionId: dd-eu ... Plumbery will only prepare nodes that have been configured for it. The example below demonstrates how this can be done for multiple docker containers:: # some docker resources - docker: domain: *vdc1 ethernet: *containers nodes: - docker1: prepare: &docker - run prepare.update.sh - run prepare.docker.sh - docker2: prepare: *docker - docker3: prepare: *docker In the real life when you have to prepare any appliance, you need to be close to the stuff and to touch it. This is the same for virtual fittings. This polisher has the need to communicate directly with target nodes over the network. This connectivity can become quite complicated because of the potential mix of private and public networks, firewalls, etc. To stay safe plumbery enforces a simple beachheading model, where network connectivity with end nodes is a no brainer. This model is based on predefined network addresses for plumbery itself, as in the snippet below:: --- # Frankfurt in Europe locationId: EU6 regionId: dd-eu # network subnets are 10.1.x.y prepare: - beachhead: 10.1.3.4 Here nodes at EU6 will be prepared only if the machine that is executing plumbery has the adress 10.1.3.4. In other cases, plumbery will state that the location is out of reach. """ def upgrade_vmware_tools(self, node): """ Upgrade VMware tools on target node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` """ if self.engine.safeMode: return True while True: try: self.region.ex_update_vm_tools(node=node) plogging.info("- upgrading vmware tools") return True except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue if 'NO_CHANGE' in str(feedback): plogging.debug("- vmware tools is already up-to-date") return True plogging.warning("- unable to upgrade vmware tools") plogging.warning(str(feedback)) return False def _apply_prepares(self, node, steps): """ Does the actual job over SSH :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param steps: the various steps of the preparing :type steps: ``list`` of ``dict`` :return: ``True`` if everything went fine, ``False`` otherwise :rtype: ``bool`` """ if node is None or node.state != NodeState.RUNNING: plogging.warning("- skipped - node is not running") return False # select the address to use if len(node.public_ips) > 0: target_ip = node.public_ips[0] elif node.extra['ipv6']: target_ip = node.extra['ipv6'] else: target_ip = node.private_ips[0] # use libcloud to communicate with remote nodes session = SSHClient(hostname=target_ip, port=22, username=self.user, password=self.secret, key_files=self.key_files, timeout=10) repeats = 0 while True: try: session.connect() break except Exception as feedback: repeats += 1 if repeats > 5: plogging.error( "Error: can not connect to '{}'!".format(target_ip)) plogging.error("- failed to connect") return False plogging.debug(str(feedback)) plogging.debug( "- connection {} failed, retrying".format(repeats)) time.sleep(10) continue while True: try: if self.engine.safeMode: plogging.info( "- skipped - no ssh interaction in safe mode") else: for step in steps: plogging.info('- {}'.format(step['description'])) step['genius'].run(node, session) except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue plogging.error("Error: unable to prepare '{}' at '{}'!".format( node.name, target_ip)) plogging.error(str(feedback)) plogging.error("- failed") result = False else: result = True break try: session.close() except: pass return result def _get_prepares(self, node, settings, container): """ Defines the set of actions to be done on a node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param settings: the fittings plan for this node :type settings: ``dict`` :param container: the container of this node :type container: :class:`plumbery.PlumberyInfrastructure` :return: a list of actions to be performed, and related descriptions :rtype: a ``list`` of `{ 'description': ..., 'genius': ... }`` """ if not isinstance(settings, dict): return [] environment = PlumberyNodeContext(node=node, container=container, context=self.facility) prepares = [] for key_file in self.key_files: try: path = os.path.expanduser(key_file) with open(path) as stream: key = stream.read() stream.close() prepares.append({ 'description': 'deploy SSH public key', 'genius': SSHKeyDeployment(key=key) }) except IOError: plogging.warning("no ssh key in {}".format(key_file)) if ('prepare' in settings and isinstance(settings['prepare'], list) and len(settings['prepare']) > 0): plogging.info('- using prepare commands') for script in settings['prepare']: tokens = script.split(' ') if len(tokens) == 1: tokens.insert(0, 'run') if tokens[0] in ['run', 'run_raw']: # send and run a script script = tokens[1] if len(tokens) > 2: args = tokens[2:] else: args = [] plogging.debug("- {} {} {}".format(tokens[0], script, ' '.join(args))) try: with open(script) as stream: text = stream.read() if (tokens[0] == 'run' and PlumberyText.could_expand(text)): plogging.debug( "- expanding script '{}'".format(script)) text = PlumberyText.expand_string( text, environment) if len(text) > 0: plogging.info("- running '{}'".format(script)) prepares.append({ 'description': ' '.join(tokens), 'genius': ScriptDeployment(script=text, args=args, name=script) }) else: plogging.error( "- script '{}' is empty".format(script)) except IOError: plogging.error( "- unable to read script '{}'".format(script)) elif tokens[0] in ['put', 'put_raw']: # send a file file = tokens[1] if len(tokens) > 2: destination = tokens[2] else: destination = './' + file plogging.debug("- {} {} {}".format(tokens[0], file, destination)) try: with open(file) as stream: content = stream.read() if (tokens[0] == 'put' and PlumberyText.could_expand(content)): plogging.debug( "- expanding file '{}'".format(file)) content = PlumberyText.expand_string( content, environment) plogging.info("- putting file '{}'".format(file)) prepares.append({ 'description': ' '.join(tokens), 'genius': FileContentDeployment(content=content, target=destination) }) except IOError: plogging.error( "- unable to read file '{}'".format(file)) else: # echo a sensible message eventually if tokens[0] == 'echo': tokens.pop(0) message = ' '.join(tokens) message = PlumberyText.expand_string(message, environment) plogging.info("- {}".format(message)) if ('cloud-config' in settings and isinstance(settings['cloud-config'], dict) and len(settings['cloud-config']) > 0): plogging.info('- using cloud-config') # mandatory, else cloud-init will not consider user-data plogging.debug('- preparing meta-data') meta_data = 'instance_id: dummy\n' destination = '/var/lib/cloud/seed/nocloud-net/meta-data' prepares.append({ 'description': 'put meta-data', 'genius': FileContentDeployment(content=meta_data, target=destination) }) plogging.debug('- preparing user-data') expanded = PlumberyText.expand_string(settings['cloud-config'], environment) user_data = '#cloud-config\n' + expanded plogging.debug(user_data) destination = '/var/lib/cloud/seed/nocloud-net/user-data' prepares.append({ 'description': 'put user-data', 'genius': FileContentDeployment(content=user_data, target=destination) }) plogging.debug('- preparing remote install of cloud-init') script = 'prepare.cloud-init.sh' try: path = os.path.dirname(__file__) + '/' + script with open(path) as stream: text = stream.read() if text: prepares.append({ 'description': 'run ' + script, 'genius': ScriptDeployment(script=text, name=script) }) except IOError: raise PlumberyException( "Error: cannot read '{}'".format(script)) plogging.debug('- preparing reboot to trigger cloud-init') prepares.append({ 'description': 'reboot node', 'genius': RebootDeployment(container=container) }) return prepares def go(self, engine): """ Starts the prepare process :param engine: access to global parameters and functions :type engine: :class:`plumbery.PlumberyEngine` """ super(PreparePolisher, self).go(engine) self.report = [] self.user = engine.get_shared_user() self.secret = engine.get_shared_secret() self.key_files = engine.get_shared_key_files() if 'key' in self.settings: key = self.settings['key'] file = os.path.expanduser(key) if os.path.isfile(file): plogging.debug("- using shared key {}".format(key)) if self.key_files is None: self.key_files = [key] else: self.key_files.insert(0, key) else: plogging.error("Error: missing file {}".format(key)) def move_to(self, facility): """ Checks if we can beachhead at this facility :param facility: access to local parameters and functions :type facility: :class:`plumbery.PlumberyFacility` This function lists all addresses of the computer that is running plumbery. If there is at least one routable IPv6 address, then it assumes that communication with nodes is possible. If no suitable IPv6 address can be found, then plumbery falls back to IPv4. Beachheading is granted only if the address of the computer running plumbery matches the fitting parameter ``beachhead``. """ self.facility = facility self.region = facility.region self.nodes = PlumberyNodes(facility) self.beachheading = False try: self.addresses = [] for interface in netifaces.interfaces(): addresses = netifaces.ifaddresses(interface) if netifaces.AF_INET in addresses.keys(): for address in addresses[netifaces.AF_INET]: # strip local loop if address['addr'].startswith('127.0.0.1'): continue self.addresses.append(address['addr']) if netifaces.AF_INET6 in addresses.keys(): for address in addresses[netifaces.AF_INET6]: # strip local loop if address['addr'].startswith('::1'): continue # strip local link addresses if address['addr'].startswith('fe80::'): continue # we have a routable ipv6, so let's go self.beachheading = True except Exception as feedback: plogging.error(str(feedback)) for item in self.facility.get_setting('prepare', []): if not isinstance(item, dict): continue if 'beachhead' not in item.keys(): continue if item['beachhead'] in self.addresses: self.beachheading = True break if self.beachheading: plogging.debug("- beachheading at '{}'".format( self.facility.get_setting('locationId'))) else: plogging.debug("- not beachheading at '{}'".format( self.facility.get_setting('locationId'))) def shine_node(self, node, settings, container): """ prepares a node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param settings: the fittings plan for this node :type settings: ``dict`` :param container: the container of this node :type container: :class:`plumbery.PlumberyInfrastructure` """ plogging.info("Preparing node '{}'".format(settings['name'])) if node is None: plogging.error("- not found") return timeout = 300 tick = 6 while node.extra['status'].action == 'START_SERVER': time.sleep(tick) node = self.nodes.get_node(node.name) timeout -= tick if timeout < 0: break if node.state != NodeState.RUNNING: plogging.error("- skipped - node is not running") return self.upgrade_vmware_tools(node) prepares = self._get_prepares(node, settings, container) if len(prepares) < 1: plogging.info('- nothing to do') self.report.append( {node.name: { 'status': 'skipped - nothing to do' }}) return if len(node.public_ips) > 0: plogging.info("- node is reachable at '{}'".format( node.public_ips[0])) elif not self.beachheading: plogging.error('- node is unreachable') self.report.append({node.name: {'status': 'unreachable'}}) return descriptions = [] for item in prepares: descriptions.append(item['description']) if self._apply_prepares(node, prepares): self.report.append( {node.name: { 'status': 'completed', 'prepares': descriptions }}) else: self.report.append( {node.name: { 'status': 'failed', 'prepares': descriptions }}) def reap(self): """ Reports on preparing """ if 'output' not in self.settings: return fileName = self.settings['output'] plogging.info("Reporting on preparations in '{}'".format(fileName)) with open(fileName, 'w') as stream: stream.write(yaml.dump(self.report, default_flow_style=False)) stream.close()
class SpitPolisher(PlumberyPolisher): """ Finalizes the setup of fittings This polisher looks at each node in sequence, and adjust settings according to fittings plan. This is covering various features that can not be set during the creation of nodes, such as: - number of CPU - quantity of RAM - monitoring """ def move_to(self, facility): """ Moves to another API endpoint :param facility: access to local parameters and functions :type facility: :class:`plumbery.PlumberyFacility` """ self.facility = facility self.region = facility.region self.nodes = PlumberyNodes(facility) def shine_container(self, container): """ Rubs a container until it shines :param container: the container to be polished :type container: :class:`plumbery.PlumberyInfrastructure` This is where the hard work is done. You have to override this function in your own polisher. Note that you can compare the reality versus the theoritical settings if you want. """ logging.info("Spitting at blueprint '{}'".format( container.blueprint['target'])) if container.network is None: logging.info("- aborted - no network here") return logging.info("- waiting for nodes to be deployed") names = self.nodes.list_nodes(container.blueprint) for name in names: while True: node = self.nodes.get_node(name) if node is None: logging.info("- aborted - missing node '{}'".format(name)) return if node.extra['status'].action is None: break if (node is not None and node.extra['status'].failure_reason is not None): logging.info("- aborted - failed deployment " "of node '{}'".format(name)) return time.sleep(20) logging.info("- nodes have been deployed") container._build_firewall_rules() container._build_balancer() def set_node_compute(self, node, cpu, memory): """ Sets compute capability :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param cpu: the cpu specification :type cpu: ``DimensionDataServerCpuSpecification`` :param memory: the memory size, expressed in Giga bytes :type memory: ``int`` """ changed = False if cpu is not None and 'cpu' in node.extra: if int(cpu.cpu_count) != int(node.extra['cpu'].cpu_count): logging.info("- changing to {} cpu".format( cpu.cpu_count)) changed = True if int(cpu.cores_per_socket) != int(node.extra['cpu'].cores_per_socket): logging.info("- changing to {} core(s) per socket".format( cpu.cores_per_socket)) changed = True if cpu.performance != node.extra['cpu'].performance: logging.info("- changing to '{}' cpu performance".format( cpu.performance.lower())) changed = True if memory is not None and 'memoryMb' in node.extra: if memory != int(node.extra['memoryMb']/1024): logging.info("- changing to {} GB memory".format( memory)) changed = True if not changed: logging.debug("- no change in compute") return if self.engine.safeMode: logging.info("- skipped - safe mode") return while True: try: self.region.ex_reconfigure_node( node=node, memory_gb=memory, cpu_count=cpu.cpu_count, cores_per_socket=cpu.cores_per_socket, cpu_performance=cpu.performance) logging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue logging.info("- unable to reconfigure node") logging.error(str(feedback)) break def change_node_disk_size(self, node, id, size): """ Changes an existing virtual disk :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param id: the disk unique identifier, as reported by the API :type id: ``str`` :param size: the disk size, expressed in Giga bytes :type size: ``int`` """ if self.engine.safeMode: logging.info("- skipped - safe mode") return while True: try: self.region.ex_change_storage_size( node=node, disk_id=id, size=size) logging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue logging.info("- unable to change disk size to {}GB" .format(size)) logging.error(str(feedback)) break def change_node_disk_speed(self, node, id, speed): """ Changes an existing virtual disk :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param id: the disk unique identifier, as reported by the API :type id: ``str`` :param speed: storage type, either 'standard', 'highperformance' or 'economy' :type speed: ``str`` """ if self.engine.safeMode: logging.info("- skipped - safe mode") return while True: try: self.region.ex_change_storage_speed( node=node, disk_id=id, speed=speed) logging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue logging.info("- unable to change disk to '{}'" .format(speed)) logging.error(str(feedback)) break def set_node_disk(self, node, id, size, speed='standard'): """ Sets a virtual disk :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param id: the disk id, starting at 0 and growing :type id: ``int`` :param size: the disk size, expressed in Giga bytes :type size: ``int`` :param speed: storage type, either 'standard', 'highperformance' or 'economy' :type speed: ``str`` """ if size < 1: logging.info("- minimum disk size is 1 GB") return if size > 1000: logging.info("- disk size cannot exceed 1000 GB") return if speed not in ['standard', 'highperformance', 'economy']: logging.info("- disk speed should be either 'standard' " "or 'highperformance' or 'economy'") return if 'disks' in node.extra: for disk in node.extra['disks']: if disk['scsiId'] == id: changed = False if disk['size'] > size: logging.info("- disk shrinking could break the node") logging.info("- skipped - disk {} will not be reduced" .format(id)) if disk['size'] < size: logging.info("- expanding disk {} to {} GB" .format(id, size)) self.change_node_disk_size(node, disk['id'], size) changed = True if disk['speed'].lower() != speed.lower(): logging.info("- changing disk {} to '{}'" .format(id, speed)) self.change_node_disk_speed(node, disk['id'], speed) changed = True if not changed: logging.debug("- no change in disk {}".format(id)) return logging.info("- adding {} GB '{}' disk".format( size, speed)) if self.engine.safeMode: logging.info("- skipped - safe mode") return while True: try: self.region.ex_add_storage_to_node( node=node, amount=size, speed=speed.upper()) logging.info("- in progress") except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue if 'Please try again later' in str(feedback): time.sleep(10) continue logging.info("- unable to add disk {} GB '{}'" .format(size, speed)) logging.error(str(feedback)) break def shine_node(self, node, settings, container): """ Finalizes setup of one node :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param settings: the fittings plan for this node :type settings: ``dict`` :param container: the container of this node :type container: :class:`plumbery.PlumberyInfrastructure` """ logging.info("Spitting at node '{}'".format(settings['name'])) if node is None: logging.info("- not found") return cpu = None if 'cpu' in settings: tokens = str(settings['cpu']).split(' ') if len(tokens) < 2: tokens.append('1') if len(tokens) < 3: tokens.append('standard') if (int(tokens[0]) < 1 or int(tokens[0]) > 32): logging.info("- cpu should be between 1 and 32") elif (int(tokens[1]) < 1 or int(tokens[1]) > 2): logging.info("- core per cpu should be either 1 or 2") elif tokens[2].upper() not in ('STANDARD', 'HIGHPERFORMANCE'): logging.info("- cpu speed should be either 'standard'" " or 'highspeed'") else: logging.debug("- setting compute {}".format(' '.join(tokens))) cpu = DimensionDataServerCpuSpecification( cpu_count=tokens[0], cores_per_socket=tokens[1], performance=tokens[2].upper()) memory = None if 'memory' in settings: memory = int(settings['memory']) if memory < 1 or memory > 256: logging.info("- memory should be between 1 and 256") memory = None else: logging.debug("- setting {} GB of memory".format( memory)) self.set_node_compute(node, cpu, memory) if 'disks' in settings: for item in settings['disks']: logging.debug("- setting disk {}".format(item)) attributes = item.split() if len(attributes) < 2: logging.info("- malformed disk attributes;" " provide disk id and size in GB, e.g., 1 50;" " add disk type if needed, e.g., economy") elif len(attributes) < 3: id = int(attributes[0]) size = int(attributes[1]) speed = 'standard' else: id = int(attributes[0]) size = int(attributes[1]) speed = attributes[2] self.set_node_disk(node, id, size, speed) if 'monitoring' in settings: self.nodes._start_monitoring(node, settings['monitoring']) if 'glue' in settings: container._attach_node(node, settings['glue']) container._add_to_pool(node)