def load_formation_from_etcd(self, username, formation_name): f = Formation(username,formation_name) app_list = json.loads(json.loads( self.etcd.get_key('/formations/{username}/{formation_name}'.format( username=username, formation_name=formation_name)))) for app in app_list: # If our host doesn't support swapping we're going to get some garbage # message in here if "WARNING" in app['container_id']: app['container_id'] = app['container_id'].replace("WARNING: Your "\ "kernel does not support memory swap capabilities. Limitation discarded.\n","") #Message changed in docker 0.8.0 app['container_id'] = app['container_id'].replace("WARNING: WARNING:"\ "Your kernel does not support swap limit capabilities. Limitation "\ "discarded.\n","") app['container_id'].strip('\n') # Set volumes if needed volumes = None if app['volumes']: self.logger.info("Setting volumes to: " + ''.join(app['volumes'])) volumes = app['volumes'] f.add_app(app['container_id'], app['hostname'], app['cpu_shares'], app['ram'], app['port_list'], app['ssh_port'], 22, app['host_server'], volumes) # Return fully parsed and populated formation object return f
def test_saveFormationToEtcd(self): logger = logging.getLogger() stream = logging.StreamHandler(sys.stdout) stream.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') stream.setFormatter(formatter) logger.addHandler(stream) manager = Manager(logger) expected_text = '[{"username": "******", "cpu-shares": 102, '\ '"ram": 100, "hostname": "app01", "ports": [{"host_port": 8080, '\ '"container_port": 8080}], "host_server": "dlceph02"}]' username = '******' formation_name = 'test_formation' f = Formation(username, formation_name) f.add_app(username, 'app01', 102, 100, [{ "host_port": 8080, "container_port": 8080 }], 'dlceph02') manager.save_formation_to_etcd(f) etcd_ret = manager.etcd.get_key('{username}/{hostname}'.format( username=username, hostname=formation_name)) logger.debug("Etcd_ret: %s" % etcd_ret) logger.debug("Expected text: %s" % expected_text) self.assertEquals(etcd_ret, expected_text)
def load_formation_from_etcd(self, username, formation_name): f = Formation(username, formation_name) app_list = json.loads( json.loads( self.etcd.get_key( '/formations/{username}/{formation_name}'.format( username=username, formation_name=formation_name)))) for app in app_list: # If our host doesn't support swapping we're going to get some garbage # message in here if "WARNING" in app['container_id']: app['container_id'] = app['container_id'].replace("WARNING: Your "\ "kernel does not support memory swap capabilities. Limitation discarded.\n","") #Message changed in docker 0.8.0 app['container_id'] = app['container_id'].replace("WARNING: WARNING:"\ "Your kernel does not support swap limit capabilities. Limitation "\ "discarded.\n","") app['container_id'].strip('\n') # Set volumes if needed volumes = None if app['volumes']: self.logger.info("Setting volumes to: " + ''.join(app['volumes'])) volumes = app['volumes'] f.add_app(app['container_id'], app['hostname'], app['cpu_shares'], app['ram'], app['port_list'], app['ssh_port'], 22, app['host_server'], volumes) # Return fully parsed and populated formation object return f
def test_saveFormationToEtcd(self): logger = logging.getLogger() stream = logging.StreamHandler(sys.stdout) stream.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') stream.setFormatter(formatter) logger.addHandler(stream) manager = Manager(logger) expected_text = '[{"username": "******", "cpu-shares": 102, '\ '"ram": 100, "hostname": "app01", "ports": [{"host_port": 8080, '\ '"container_port": 8080}], "host_server": "dlceph02"}]' username = '******' formation_name = 'test_formation' f = Formation(username, formation_name) f.add_app(username, 'app01', 102, 100, [{"host_port":8080, "container_port":8080}], 'dlceph02') manager.save_formation_to_etcd(f) etcd_ret = manager.etcd.get_key('{username}/{hostname}'.format(username=username, hostname=formation_name)) logger.debug("Etcd_ret: %s" % etcd_ret) logger.debug("Expected text: %s" % expected_text) self.assertEquals(etcd_ret, expected_text)
def create_containers(self, user, number, formation_name, cpu_shares, ram, port_list, hostname_scheme, volume_list, force_host_server=None): f = Formation(user, formation_name) # Convert ram to bytes from MB ram = ram * 1024 * 1024 # Get the cluster machines on each creation cluster_list = self.get_docker_cluster() circular_cluster_list = CircularList(self.order_cluster_by_load(cluster_list)) # Loop for the requested amount of containers to be created for i in range(1, number+1): # [{"host_port":ssh_host_port, "container_port":ssh_container_port}] ssh_host_port = 9022 + i ssh_container_port = 22 host_server = circular_cluster_list[i].hostname # We are being asked to overwrite this if force_host_server: host_server = force_host_server validated_ports = [] while self.check_port_used(host_server, ssh_host_port): ssh_host_port = ssh_host_port +1 for port in port_list: self.logger.info("Checking if port {port} on {host} is in use".format( port=port, host=host_server)) if ':' in port: ports = port.split(':') # Only check if the host port is free. The container port should be free while self.check_port_used(host_server, ports[0]): ports[0] = int(ports[0]) + 1 # Add this to the validated port list validated_ports.append('{host_port}:{container_port}'.format( host_port = str(ports[0]), container_port = str(ports[1]))) else: while self.check_port_used(host_server, port): port = int(port) + 1 validated_ports.append(str(port)) self.logger.info('Adding app to formation {formation_name}: {hostname}{number} cpu_shares={cpu} ' 'ram={ram} ports={ports} host_server={host_server}'.format(formation_name=formation_name, hostname=hostname_scheme, number=str(i).zfill(3), cpu=cpu_shares, ram=ram, ports=validated_ports, host_server=host_server)) f.add_app(None, '{hostname}{number}'.format(hostname=hostname_scheme, number=str(i).zfill(3)), cpu_shares, ram, validated_ports, ssh_host_port, ssh_container_port, circular_cluster_list[i].hostname, volume_list) # Lets get this party started for app in f.application_list: self.start_application(app) self.logger.info("Sleeping 2 seconds while the container starts") time.sleep(2) self.bootstrap_application(app) self.logger.info("Saving the formation to ETCD") self.save_formation_to_etcd(f)
def create_containers(self, user, number, formation_name, cpu_shares, ram, port_list, hostname_scheme, volume_list, docker_image, force_host_server=None): f = Formation(user, formation_name) # Convert ram to bytes from MB ram = ram * 1024 * 1024 # Get the cluster machines on each creation cluster_list = self.get_docker_cluster() circular_cluster_list = CircularList( self.order_cluster_by_load(cluster_list)) # Loop for the requested amount of containers to be created for i in range(1, number + 1): # [{"host_port":ssh_host_port, "container_port":ssh_container_port}] ssh_host_port = 9022 + i ssh_container_port = 22 host_server = circular_cluster_list[i].hostname hostname = '{hostname}{number}'.format(hostname=hostname_scheme, number=str(i).zfill(3)) # First check if we can add this host to salt. If not exit with -1 if self.check_salt_key_used(hostname): self.logger.error( 'Salt key is already taken for {hostname}'.format( hostname=hostname)) sys.exit(-1) # We are being asked to overwrite this if force_host_server: host_server = force_host_server validated_ports = [] while self.check_port_used(host_server, ssh_host_port): ssh_host_port = ssh_host_port + 1 for port in port_list: self.logger.info( "Checking if port {port} on {host} is in use".format( port=port, host=host_server)) if ':' in port: ports = port.split(':') # Only check if the host port is free. The container port should be free while self.check_port_used(host_server, ports[0]): ports[0] = int(ports[0]) + 1 # Add this to the validated port list validated_ports.append( '{host_port}:{container_port}'.format( host_port=str(ports[0]), container_port=str(ports[1]))) else: while self.check_port_used(host_server, port): port = int(port) + 1 validated_ports.append(str(port)) self.logger.info( 'Adding app to formation {formation_name}: {hostname} cpu_shares={cpu} ' 'ram={ram} ports={ports} host_server={host_server} docker_image={docker_image}' .format(formation_name=formation_name, hostname=hostname, cpu=cpu_shares, ram=ram, ports=validated_ports, host_server=host_server, docker_image=docker_image)) f.add_app(None, '{hostname}'.format(hostname=hostname), cpu_shares, ram, validated_ports, ssh_host_port, ssh_container_port, host_server, docker_image, volume_list) # Lets get this party started for app in f.application_list: self.start_application(app) #self.logger.info("Sleeping 2 seconds while the container starts") #time.sleep(2) #self.bootstrap_application(app) self.logger.info("Saving the formation to ETCD") self.save_formation_to_etcd(f)