def load_formation_from_etcd(self, username, formation_name):
f = Formation(username,formation_name)
app_list = json.loads(json.loads(
self.etcd.get_key('/formations/{username}/{formation_name}'.format(
username=username, formation_name=formation_name))))
for app in app_list:
# If our host doesn't support swapping we're going to get some garbage
# message in here
if "WARNING" in app['container_id']:
app['container_id'] = app['container_id'].replace("WARNING: Your "\
"kernel does not support memory swap capabilities. Limitation discarded.\n","")
#Message changed in docker 0.8.0
app['container_id'] = app['container_id'].replace("WARNING: WARNING:"\
"Your kernel does not support swap limit capabilities. Limitation "\
"discarded.\n","")
app['container_id'].strip('\n')
# Set volumes if needed
volumes = None
if app['volumes']:
self.logger.info("Setting volumes to: " + ''.join(app['volumes']))
volumes = app['volumes']
f.add_app(app['container_id'], app['hostname'], app['cpu_shares'],
app['ram'], app['port_list'], app['ssh_port'], 22, app['host_server'], volumes)
# Return fully parsed and populated formation object
return f
def test_saveFormationToEtcd(self):
logger = logging.getLogger()
stream = logging.StreamHandler(sys.stdout)
stream.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
stream.setFormatter(formatter)
logger.addHandler(stream)
manager = Manager(logger)
expected_text = '[{"username": "******", "cpu-shares": 102, '\
'"ram": 100, "hostname": "app01", "ports": [{"host_port": 8080, '\
'"container_port": 8080}], "host_server": "dlceph02"}]'
username = '******'
formation_name = 'test_formation'
f = Formation(username, formation_name)
f.add_app(username, 'app01', 102, 100, [{
"host_port": 8080,
"container_port": 8080
}], 'dlceph02')
manager.save_formation_to_etcd(f)
etcd_ret = manager.etcd.get_key('{username}/{hostname}'.format(
username=username, hostname=formation_name))
logger.debug("Etcd_ret: %s" % etcd_ret)
logger.debug("Expected text: %s" % expected_text)
self.assertEquals(etcd_ret, expected_text)
def load_formation_from_etcd(self, username, formation_name):
f = Formation(username, formation_name)
app_list = json.loads(
json.loads(
self.etcd.get_key(
'/formations/{username}/{formation_name}'.format(
username=username, formation_name=formation_name))))
for app in app_list:
# If our host doesn't support swapping we're going to get some garbage
# message in here
if "WARNING" in app['container_id']:
app['container_id'] = app['container_id'].replace("WARNING: Your "\
"kernel does not support memory swap capabilities. Limitation discarded.\n","")
#Message changed in docker 0.8.0
app['container_id'] = app['container_id'].replace("WARNING: WARNING:"\
"Your kernel does not support swap limit capabilities. Limitation "\
"discarded.\n","")
app['container_id'].strip('\n')
# Set volumes if needed
volumes = None
if app['volumes']:
self.logger.info("Setting volumes to: " +
''.join(app['volumes']))
volumes = app['volumes']
f.add_app(app['container_id'], app['hostname'], app['cpu_shares'],
app['ram'], app['port_list'], app['ssh_port'], 22,
app['host_server'], volumes)
# Return fully parsed and populated formation object
return f
def test_saveFormationToEtcd(self):
logger = logging.getLogger()
stream = logging.StreamHandler(sys.stdout)
stream.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
stream.setFormatter(formatter)
logger.addHandler(stream)
manager = Manager(logger)
expected_text = '[{"username": "******", "cpu-shares": 102, '\
'"ram": 100, "hostname": "app01", "ports": [{"host_port": 8080, '\
'"container_port": 8080}], "host_server": "dlceph02"}]'
username = '******'
formation_name = 'test_formation'
f = Formation(username, formation_name)
f.add_app(username, 'app01', 102, 100, [{"host_port":8080, "container_port":8080}], 'dlceph02')
manager.save_formation_to_etcd(f)
etcd_ret = manager.etcd.get_key('{username}/{hostname}'.format(username=username, hostname=formation_name))
logger.debug("Etcd_ret: %s" % etcd_ret)
logger.debug("Expected text: %s" % expected_text)
self.assertEquals(etcd_ret, expected_text)
def create_containers(self, user, number, formation_name,
cpu_shares, ram, port_list, hostname_scheme, volume_list,
force_host_server=None):
f = Formation(user, formation_name)
# Convert ram to bytes from MB
ram = ram * 1024 * 1024
# Get the cluster machines on each creation
cluster_list = self.get_docker_cluster()
circular_cluster_list = CircularList(self.order_cluster_by_load(cluster_list))
# Loop for the requested amount of containers to be created
for i in range(1, number+1):
# [{"host_port":ssh_host_port, "container_port":ssh_container_port}]
ssh_host_port = 9022 + i
ssh_container_port = 22
host_server = circular_cluster_list[i].hostname
# We are being asked to overwrite this
if force_host_server:
host_server = force_host_server
validated_ports = []
while self.check_port_used(host_server, ssh_host_port):
ssh_host_port = ssh_host_port +1
for port in port_list:
self.logger.info("Checking if port {port} on {host} is in use".format(
port=port, host=host_server))
if ':' in port:
ports = port.split(':')
# Only check if the host port is free. The container port should be free
while self.check_port_used(host_server, ports[0]):
ports[0] = int(ports[0]) + 1
# Add this to the validated port list
validated_ports.append('{host_port}:{container_port}'.format(
host_port = str(ports[0]),
container_port = str(ports[1])))
else:
while self.check_port_used(host_server, port):
port = int(port) + 1
validated_ports.append(str(port))
self.logger.info('Adding app to formation {formation_name}: {hostname}{number} cpu_shares={cpu} '
'ram={ram} ports={ports} host_server={host_server}'.format(formation_name=formation_name,
hostname=hostname_scheme, number=str(i).zfill(3), cpu=cpu_shares, ram=ram,
ports=validated_ports, host_server=host_server))
f.add_app(None, '{hostname}{number}'.format(hostname=hostname_scheme,
number=str(i).zfill(3)), cpu_shares, ram, validated_ports, ssh_host_port,
ssh_container_port, circular_cluster_list[i].hostname, volume_list)
# Lets get this party started
for app in f.application_list:
self.start_application(app)
self.logger.info("Sleeping 2 seconds while the container starts")
time.sleep(2)
self.bootstrap_application(app)
self.logger.info("Saving the formation to ETCD")
self.save_formation_to_etcd(f)
def create_containers(self,
user,
number,
formation_name,
cpu_shares,
ram,
port_list,
hostname_scheme,
volume_list,
docker_image,
force_host_server=None):
f = Formation(user, formation_name)
# Convert ram to bytes from MB
ram = ram * 1024 * 1024
# Get the cluster machines on each creation
cluster_list = self.get_docker_cluster()
circular_cluster_list = CircularList(
self.order_cluster_by_load(cluster_list))
# Loop for the requested amount of containers to be created
for i in range(1, number + 1):
# [{"host_port":ssh_host_port, "container_port":ssh_container_port}]
ssh_host_port = 9022 + i
ssh_container_port = 22
host_server = circular_cluster_list[i].hostname
hostname = '{hostname}{number}'.format(hostname=hostname_scheme,
number=str(i).zfill(3))
# First check if we can add this host to salt. If not exit with -1
if self.check_salt_key_used(hostname):
self.logger.error(
'Salt key is already taken for {hostname}'.format(
hostname=hostname))
sys.exit(-1)
# We are being asked to overwrite this
if force_host_server:
host_server = force_host_server
validated_ports = []
while self.check_port_used(host_server, ssh_host_port):
ssh_host_port = ssh_host_port + 1
for port in port_list:
self.logger.info(
"Checking if port {port} on {host} is in use".format(
port=port, host=host_server))
if ':' in port:
ports = port.split(':')
# Only check if the host port is free. The container port should be free
while self.check_port_used(host_server, ports[0]):
ports[0] = int(ports[0]) + 1
# Add this to the validated port list
validated_ports.append(
'{host_port}:{container_port}'.format(
host_port=str(ports[0]),
container_port=str(ports[1])))
else:
while self.check_port_used(host_server, port):
port = int(port) + 1
validated_ports.append(str(port))
self.logger.info(
'Adding app to formation {formation_name}: {hostname} cpu_shares={cpu} '
'ram={ram} ports={ports} host_server={host_server} docker_image={docker_image}'
.format(formation_name=formation_name,
hostname=hostname,
cpu=cpu_shares,
ram=ram,
ports=validated_ports,
host_server=host_server,
docker_image=docker_image))
f.add_app(None, '{hostname}'.format(hostname=hostname), cpu_shares,
ram, validated_ports, ssh_host_port, ssh_container_port,
host_server, docker_image, volume_list)
# Lets get this party started
for app in f.application_list:
self.start_application(app)
#self.logger.info("Sleeping 2 seconds while the container starts")
#time.sleep(2)
#self.bootstrap_application(app)
self.logger.info("Saving the formation to ETCD")
self.save_formation_to_etcd(f)
