def test_update_headers(self):
sample_headers = {
'X-Docker-Locale': 'en-US',
}
def f(self, headers=None):
return headers
client = APIClient()
client._auth_configs = {}
g = update_headers(f)
assert g(client, headers=None) is None
assert g(client, headers={}) == {}
assert g(client, headers={'Content-type': 'application/json'}) == {
'Content-type': 'application/json',
}
client._auth_configs = {
'HttpHeaders': sample_headers
}
assert g(client, headers=None) == sample_headers
assert g(client, headers={}) == sample_headers
assert g(client, headers={'Content-type': 'application/json'}) == {
'Content-type': 'application/json',
'X-Docker-Locale': 'en-US',
}
def test_kwargs_from_env_tls(self):
os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376',
DOCKER_CERT_PATH=TEST_CERT_DIR,
DOCKER_TLS_VERIFY='1')
kwargs = kwargs_from_env(assert_hostname=False)
self.assertEqual('https://192.168.59.103:2376', kwargs['base_url'])
self.assertTrue('ca.pem' in kwargs['tls'].ca_cert)
self.assertTrue('cert.pem' in kwargs['tls'].cert[0])
self.assertTrue('key.pem' in kwargs['tls'].cert[1])
self.assertEqual(False, kwargs['tls'].assert_hostname)
self.assertTrue(kwargs['tls'].verify)
try:
client = APIClient(**kwargs)
self.assertEqual(kwargs['base_url'], client.base_url)
self.assertEqual(kwargs['tls'].ca_cert, client.verify)
self.assertEqual(kwargs['tls'].cert, client.cert)
except TypeError as e:
self.fail(e)
def test_kwargs_from_env_tls(self):
os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376',
DOCKER_CERT_PATH=TEST_CERT_DIR,
DOCKER_TLS_VERIFY='1')
kwargs = kwargs_from_env(assert_hostname=False)
assert 'https://192.168.59.103:2376' == kwargs['base_url']
assert 'ca.pem' in kwargs['tls'].ca_cert
assert 'cert.pem' in kwargs['tls'].cert[0]
assert 'key.pem' in kwargs['tls'].cert[1]
assert kwargs['tls'].assert_hostname is False
assert kwargs['tls'].verify
try:
client = APIClient(**kwargs)
assert kwargs['base_url'] == client.base_url
assert kwargs['tls'].ca_cert == client.verify
assert kwargs['tls'].cert == client.cert
except TypeError as e:
self.fail(e)
def test_kwargs_from_env_tls_verify_false(self):
os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376',
DOCKER_CERT_PATH=TEST_CERT_DIR,
DOCKER_TLS_VERIFY='')
kwargs = kwargs_from_env(assert_hostname=True)
assert 'tcp://192.168.59.103:2376' == kwargs['base_url']
assert 'ca.pem' in kwargs['tls'].ca_cert
assert 'cert.pem' in kwargs['tls'].cert[0]
assert 'key.pem' in kwargs['tls'].cert[1]
assert kwargs['tls'].assert_hostname is True
assert kwargs['tls'].verify is False
parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True)
try:
client = APIClient(**kwargs)
assert parsed_host == client.base_url
assert kwargs['tls'].cert == client.cert
assert not kwargs['tls'].verify
except TypeError as e:
self.fail(e)
def cleanup(self):
client = APIClient()
# Remove unused volumes.
volumes = client.volumes({'dangling': True})
if volumes and volumes['Volumes']:
for volume in volumes['Volumes']:
try:
client.remove_volume(volume['Name'])
except Exception:
pass
# Remove unused images.
images = client.images(filters={'dangling': True})
if images:
for image in images:
try:
client.remove_image(image['Id'], force=True)
except Exception:
pass
def __init__(self, name, process_id=None):
self.name = name
self.process_id = process_id
self._docker_client = docker.from_env()
self._docker_api_client = APIClient(kwargs_from_env())
self._hosts = None
class Network(object):
"""Atomix test network."""
def __init__(self, name, process_id=None):
self.name = name
self.process_id = process_id
self._docker_client = docker.from_env()
self._docker_api_client = APIClient(kwargs_from_env())
self._hosts = None
@property
def docker_network(self):
try:
return self._docker_client.networks.get(self.name)
except docker.errors.NotFound:
raise UnknownNetworkError(self.name)
@property
def subnet(self):
return str(
self._docker_api_client.inspect_network(
self.name)['IPAM']['Config'][0]['Subnet'])
@property
def gateway(self):
return str(
self._docker_api_client.inspect_network(
self.name)['IPAM']['Config'][0]['Gateway'])
@property
def hosts(self):
if self._hosts is None:
self._hosts = self._create_hosts_iterator()
return self._hosts
def _create_hosts_iterator(self):
"""Creates a host iterator from available hosts by inspecting existing containers attached to the network."""
ips = set([self.gateway] + [
str(IPv4Interface(container['IPv4Address']).ip)
for container in self._docker_api_client.inspect_network(self.name)
['Containers'].values()
])
for host in IPv4Network(unicode(self.subnet)).hosts():
host = str(host)
if host not in ips:
yield host
def setup(self, supernet='172.18.0.0/16', subnet=None, gateway=None):
"""Sets up the network."""
def find_subnet():
docker_subnets = []
for network in self._docker_client.networks.list():
network_info = self._docker_api_client.inspect_network(
network.name)
if len(network_info['IPAM']['Config']) > 0:
docker_subnets.append(
str(network_info['IPAM']['Config'][0]['Subnet']))
for subnet in IPv4Network(
unicode(supernet)).subnets(new_prefix=24):
if str(subnet) not in docker_subnets:
return str(subnet)
raise UnknownNetworkError(
"Cannot find available subnet from supernet {}".format(
supernet))
if subnet is None:
subnet = find_subnet()
self._hosts = iter(
[str(host) for host in IPv4Network(unicode(subnet)).hosts()])
if gateway is None:
gateway = str(next(self._hosts))
ipam_pool = docker.types.IPAMPool(subnet=subnet, gateway=gateway)
ipam_config = docker.types.IPAMConfig(pool_configs=[ipam_pool])
logger.info("Creating network")
labels = {
'atomix-test': 'true',
'atomix-process': self.process_id or '',
'atomix-cluster': self.name
}
self._docker_client.networks.create(self.name,
driver='bridge',
ipam=ipam_config,
labels=labels)
def teardown(self):
"""Tears down the network."""
logger.info("Removing network")
try:
self.docker_network.remove()
except:
pass
def partition(self, local, remote=None):
"""Partitions the given local from the given remote using a bi-directional partition."""
local, remote = self._get_node(local), self._get_node(remote)
return self.bipartition(local, remote)
def unipartition(self, local, remote=None):
"""Partitions the given local from the given remote."""
local, remote = self._get_node(local), self._get_node(remote)
if remote is not None:
return self._partition(local, remote)
else:
disruptions = []
for name, ip in self._interfaces():
if name != local:
disruptions.append(self._partition(local, name))
return with_context(*disruptions)
def bipartition(self, node1, node2=None):
"""Creates a bi-directional partition between the two nodes."""
node1, node2 = self._get_node(node1), self._get_node(node2)
if node2 is not None:
return with_context(self._partition(node1, node2),
self._partition(node2, node1))
else:
disruptions = []
for name, ip in self._interfaces():
if name != node1:
disruptions.append(self.bipartition(node1, name))
return with_context(*disruptions)
def heal(self, local=None, remote=None):
"""Heals partitions."""
local, remote = self._get_node(local), self._get_node(remote)
if local is not None and remote is not None:
self._heal(local, remote)
self._heal(remote, local)
elif local is not None:
for name, ip in self._interfaces():
if name != local:
self._heal(local, name)
self._heal(name, local)
else:
for name1, ip1 in self._interfaces():
for name2, ip2 in self._interfaces():
if name1 != name2:
self._heal(name1, name2)
self._heal(name2, name1)
def partition_halves(self):
"""Partitions the network into two halves."""
disruptions = []
ips = self._interfaces()
for i in range(len(ips)):
if i % 2 == 0:
for j in range(len(ips)):
if i != j and j % 2 == 1:
disruptions.append(
self.bipartition(ips[i][0], ips[j][0]))
return with_context(*disruptions)
def partition_random(self):
"""Partitions a random node."""
node1 = self._random_interface()[0]
node2 = self._random_interface()[0]
while node1 == node2:
node2 = self._random_interface()
return with_context(self.bipartition(node1, node2))
def partition_bridge(self, node=None):
"""Partitions a node as a bridge to two sides of a cluster."""
if node is None:
node = self._random_interface()[0]
else:
node = self._get_node(node)
interfaces = self._interfaces()
disruptions = []
for i in range(len(interfaces)):
if i % 2 == 0 and interfaces[i][0] != node:
for j in range(len(interfaces)):
if i != j and j % 2 == 1 and interfaces[j][0] != node:
disruptions.append(
self.bipartition(interfaces[i][0],
interfaces[j][0]))
return with_context(*disruptions)
def partition_isolate(self, node=None):
"""Isolates the given node from all its peers."""
if node is None:
node = self._random_interface()[0]
else:
node = self._get_node(node)
disruptions = []
for name, ip in self._interfaces():
if name != node:
disruptions.append(self.bipartition(node, name))
return with_context(*disruptions)
def delay(self,
node=None,
latency=50,
jitter=10,
correlation=.75,
distribution='normal'):
"""Delays packets to the given node."""
if node is None:
return with_context(*[
self._delay(name, latency, jitter, correlation, distribution)
for name, ip in self._interfaces()
])
return self._delay(self._get_node(node), latency, jitter, correlation,
distribution)
def drop(self, node=None, probability=.02, correlation=.25):
"""Drops packets to the given node."""
if node is None:
return with_context(*[
self._drop(name, probability, correlation)
for name, ip in self._interfaces()
])
return self._drop(self._get_node(node), probability, correlation)
def reorder(self, node=None, probability=.02, correlation=.5):
"""Reorders packets to the given node."""
if node is None:
return with_context(*[
self._reorder(name, probability, correlation)
for name, ip in self._interfaces()
])
return self._reorder(self._get_node(node), probability, correlation)
def duplicate(self, node=None, probability=.005, correlation=.05):
"""Duplicates packets to the given node."""
if node is None:
return with_context(*[
self._duplicate(name, probability, correlation)
for name, ip in self._interfaces()
])
return self._duplicate(self._get_node(node), probability, correlation)
def corrupt(self, node=None, probability=.02):
"""Duplicates packets to the given node."""
if node is None:
return with_context(*[
self._corrupt(name, probability)
for name, ip in self._interfaces()
])
return self._corrupt(self._get_node(node), probability)
def restore(self, node=None):
"""Restores packets to the given node to normal order."""
if node is None:
for name, ip in self._interfaces():
self._restore(name)
else:
self._restore(self._get_node(node))
def _partition(self, local, remote):
"""Partitions the given local from the given remote."""
logger.info("Cutting off link %s->%s", local, remote)
self._run_in_container(local, 'iptables', '-A', 'INPUT', '-s',
self._get_ip(remote), '-j', 'DROP', '-w')
return with_context(lambda: self.heal(local, remote))
def _heal(self, local, remote):
"""Heals a partition from the given local to the given remote."""
logger.info("Restoring link %s->%s", local, remote)
self._run_in_container(local, 'iptables', '-D', 'INPUT', '-s',
self._get_ip(remote), '-j', 'DROP', '-w')
def _delay(self,
node,
latency=50,
jitter=10,
correlation=.75,
distribution='normal'):
"""Delays packets to the given node."""
latency, jitter, correlation = self._millize(latency), self._millize(
jitter), self._percentize(correlation)
logger.info(
"Delaying packets to %s (latency=%s, jitter=%s, correlation=%s, distribution=%s)",
node, latency, jitter, correlation, distribution)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'delay', latency, jitter,
correlation, 'distribution', distribution)
return with_context(lambda: self.restore(node))
def _drop(self, node, probability=.02, correlation=.25):
"""Drops packets to the given node."""
probability, correlation = self._percentize(
probability), self._percentize(correlation)
logger.info("Dropping packets to %s (probability=%s, correlation=%s)",
node, probability, correlation)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'loss', probability,
correlation)
return with_context(lambda: self.restore(node))
def _reorder(self, node, probability=.02, correlation=.5):
"""Reorders packets to the given node."""
probability, correlation = self._percentize(
probability), self._percentize(correlation)
logger.info(
"Reordering packets to %s (probability=%s, correlation=%s)", node,
probability, correlation)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'reorder', probability,
correlation)
return with_context(lambda: self.restore(node))
def _duplicate(self, node, probability=.005, correlation=.05):
"""Duplicates packets to the given node."""
probability, correlation = self._percentize(
probability), self._percentize(correlation)
logger.info(
"Duplicating packets to %s (probability=%s, correlation=%s)", node,
probability, correlation)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'duplicate', probability,
correlation)
return with_context(lambda: self.restore(node))
def _corrupt(self, node, probability=.02):
"""Duplicates packets to the given node."""
probability = self._percentize(probability)
logger.info("Corrupting packets to %s (probability=%s)", node,
probability)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'corrupt', probability)
return with_context(lambda: self.restore(node))
def _restore(self, node):
"""Restores packets to the given node to normal order."""
logger.info("Restoring packets to %s", node)
self._run_in_container(node, 'tc', 'qdisc', 'del', 'dev', 'eth0',
'root')
def _run_in_container(self, node, *command):
command = ' '.join([shlex_quote(str(arg)) for arg in command])
self._get_container(node).exec_run(command)
def _percentize(self, d, digits=2):
return '{}%'.format(round(d * 100, digits))
def _millize(self, ms):
return '{}ms'.format(ms)
def _interfaces(self):
"""Lists the IPs used in the network."""
containers = self._docker_api_client.inspect_network(
self.name)['Containers']
return sorted([(container['Name'],
str(IPv4Interface(container['IPv4Address']).ip))
for container in containers.values()],
key=lambda x: x[0])
def _random_interface(self):
"""Returns a random interface name, ip pair."""
return random.choice(self._interfaces())
def _get_container(self, name):
return self._docker_client.containers.get(name)
def _get_ip(self, name):
return dict(self._interfaces())[name]
def _get_node(self, name):
if name is None:
return None
try:
return '{}-{}'.format(self.name, int(name))
except ValueError:
return name
class dockerfile_creater():
def __init__(self, dockerdir=getcwd(), dockername=config.get("variable", "dockername"), registry=config.get("variable", "registry")):
"""Parameter:\n\
dockerdir: Arbeits Verzeichnis (Default: aktueles Verzeichnis)\n\
dockername: Filename des generirten Dockerfiles (Default: Dockerfile)\n\
repository: Docker Repository (Default: dockerregistry:5000)\n\
Mach der initialisirung kann <Class>.<Variable> manual gesetzt werden:\n\
.healthttp: Befehl um ein http Test auf diesen Basis-Beriebsystem zu machen"""
self.git_clean = config.get("befehle", "git_clean").split()
self.git_get = config.get("befehle", "git_get").split()
self.copy = config.get("befehle", "copy").split()
self.copydir = config.get("befehle", "copydir").split()
self.delete = config.get("befehle", "delete").split()
self.deldir = config.get("befehle", "deldir").split()
self.wget = config.get("befehle", "wget").split()
self.unzip = config.get("befehle", "unzip").split()
self.pipsearch = config.get("befehle", "pip_privet_search").split()
self.dockertag = config.get("befehle", "dockertag").split()
self.dockerpush = config.get("befehle", "dockerpush").split()
self.dockerrmi = config.get("befehle", "dockerrmi").split()
environ["DOCKER_HOST"] = config.get("variable", "deamon")
self.docker_deamon = APIClient(base_url=config.get("variable", "deamon"), version=config.get("variable", "deamon_version"))
self.pippurl = config.get("variable", "privet_pip")
self.workdir = getcwd()
self.dockerfile = []
self.copyfiles = []
self.tag = None
self.gitdir = ""
self.httpfile = ""
self.healthhttp = ""
self.registry = registry
self.dockerdir = dockerdir
self.dockername = dockername
self.ptag = config.get("variable", "product_tag")
self.otag = config.get("variable", "old_tag")
self.ttag = config.get("variable", "test_tag")
self.ltag = config.get("variable", "latest_tag")
self.ftag = config.get("variable", "fail_tag")
if len(argv) > 1:
if argv[1] == "-p":
self.production = True
else:
self.production = False
# Schreibe Config
@staticmethod
def write_config():
"""Schreibt ein Config file Beispiel"""
with open('dockerfile_generator_example.cfg', 'wb') as configfile:
config.write(configfile)
# git Methoden
def git_clean_m(self, path):
"""Aktualisirt ein Git Verzeichnis\n\
Parameter: Verzeichnis-Path des Git Repository"""
chdir(path)
call(self.git_clean)
chdir(self.workdir)
def git_get_m(self, repository):
"""Holt ein git-Repository ins Work Verzeichnis\n\
Parameter: repository URL von githup. Das root Verzeichnis wird in das Atribut gitdir uebertragen."""
chdir(self.dockerdir)
call(self.git_get + [repository])
self.gitdir = repository.split("/")[-1][:-4]
self.copyfiles.append(repository.split("/")[-1][:-4])
chdir(self.workdir)
# File Methoden in Dockerverzeichnis
def copy_in(self, pathlist, autoadd=False):
"""Methode um Dateien von Orginal Verzeichnis zum Arbeits Verzeichnis zu kopiren\n\
Parameter: Liste aus Path Elemeneten zu der zu kopirenden Datei\n\
Format: [path, path, ..., kompletter path in gewuenschten git subverzeichnis]\n\
autoadd: Programinternen Flack, wird von der add Methode benutzt, nicht manuell benutzen."""
include_dir = self.path2file(pathlist)
if autoadd:
filelist = listdir(include_dir)
for item in filelist:
if isdir(pjoin(include_dir, item)):
call(self.copydir + [pjoin(include_dir, item), self.dockerdir])
else:
call(self.copy + [pjoin(include_dir, item), self.dockerdir])
self.copyfiles = self.copyfiles + filelist
else:
if isdir(pjoin(include_dir)):
call(self.copydir + [pjoin(include_dir), self.dockerdir])
else:
call(self.copy + [pjoin(include_dir), self.dockerdir])
self.copyfiles = self.copyfiles + pathlist[-1:]
def copy_HTTP(self, url):
"""Methode um Dateien von Internet zum Arbeits Verzeichnis zu kopiren\n\
Parameter: URL zu der zu kopirenden Datei"""
chdir(self.dockerdir)
call(self.wget + [url])
chdir(self.workdir)
self.httpfile = url.split("/")[-1]
self.copyfiles.append(url.split("/")[-1])
def rm_copy_files(self, debug=False):
"""Methode um alle kopirten Orginal Dateien aus den Arbeits Verzeichnis zu loeschen\n\
Parameter: debug: behalten des generierten Dockerfiles."""
chdir(self.dockerdir)
for item in self.copyfiles:
if isdir(item):
call(self.deldir + [item])
else:
call(self.delete + [item])
if not debug:
call(self.delete + [self.dockername])
chdir(self.workdir)
# Bearbeite Dockerfile Methoden
def line_add(self, insert, file=False, nocash=False):
"""Fuegt eine Zeile oder eine Datei in Dockerfile ein\n\
Parameter: Text der Zeile oder Datei-Pfat"""
if nocash:
self.dockerfile.append("RUN echo %i\n" % (randint(1, 100000)))
if file:
fh = open(insert, "r")
for line in fh:
self.dockerfile.append("%s\n" % line)
fh.close()
else:
self.dockerfile.append("%s\n" % insert)
def line_del(self, text):
"""Loescht alle Zeilen aus den Dockerfile die einen Suchbegrif enthalten\n\
Parameter: Suchbegrif"""
templist = self.dockerfile
self.dockerfile = []
for line in templist:
if line.find(text) > -1:
continue
self.dockerfile.append(line)
def line_rewrite(self, search, text, nocash=False):
"""Ersetzt alle Zeilen aus den Dockerfile die einen Suchbegrif enthalten\n\
Parameter: search = Suchbegrif\n\
text = Text der zu ersetzenden Zeile"""
if nocash:
self.dockerfile.append("RUN echo %i\n" % (randint(1, 100000)))
templist = self.dockerfile
self.dockerfile = []
for line in templist:
if line.find(search) > -1:
self.dockerfile.append("%s\n" % text)
else:
self.dockerfile.append(line)
def add_dockerverzeichnis(self, pathlist=None, dirflack=True, delexpose=True, start=False):
"""Fuegt ein Docker Produkt ein\n\
Parameter:\n\
pathlist: Liste aus Path Elementen die zum Verzeichnis der importirten Dockerfiles fuehren\n\
Das Format der Liste ist [git-Verzeichnis, Subverzeichnis, Subverzeichnis, ...]\n\
Ist das Verzeichnis nicht in github mus githup auf False gesetzt werden\n\
In Verzeichnis wird eine Datei Names 'Dockerfile' erwartet\n\
dirflack: Flack ob die Orginaldateien kopiert werden sollen oder nicht (Default: True)\n\
ACHTUNG: enthaelt das Verzeichnis eine .dockerignore dann ist ein automatisches kopiren ueber\n\
dieses Flack nicht moeglich. Die Dateien muessen manuell mit copy_in-Methode kopiert werden\n\
delexpose: Flack um EXPOSE angaben zu loeschen (Default: True)\n\
start: Flack ob das hinzugefuegte Docker Produckt das Start-Base Docker Produkt ist (Default: False)"""
pathlist = pathlist or []
if not pathlist:
stdout.write("\n")
stdout.write("\nSubverzeichnis zum Dockerfile: \n\n")
file = stdin.readline()[:-1]
stdout.write("\n")
else:
# gitdir = pathlist[0]
file = self.path2file(pathlist + ["Dockerfile"])
fh = open(file, "r")
for line in fh:
if line.strip().startswith("FROM") and not start:
continue
if line.strip().startswith("EXPOSE") and delexpose:
continue
self.dockerfile.append(line)
fh.close()
if dirflack:
self.copy_in(pathlist, autoadd=True)
def add_uwsgi_product(self, conf_dir, initfile="production.ini", cmd="pserve /production.ini", nocash=True, version=True):
self.copy_in(conf_dir + [initfile])
modul_list = []
iniconf = ConfigParser()
iniconf.read(initfile)
modul_list.append(self.rawmodul_to_modul(iniconf['app:main']['use']))
main_modul = self.rawmodul_to_modul(iniconf['app:main']['use'])
modul_list.append(main_modul)
try:
modul_includes = iniconf['app:main']['pyramid.includes']
modul_includes = modul_includes.split("\n")
for modul in modul_includes:
modul_list.append(self.rawmodul_to_modul(modul))
except:
pass
modul_list = sorted(set(modul_list), key=modul_list.index)
try:
modul_list.remove("")
except:
pass
self.dockerfile.append("ADD %s /\n" % initfile)
self.dockerfile.append("RUN python -VV; pip -V; uname -a\n")
if nocash:
self.dockerfile.append("RUN echo %i\n" % (randint(1, 100000)))
self.dockerfile.append("RUN pip --disable-pip-version-check --no-cache-dir --no-color install %s\n" % (" ".join(modul_list)))
self.dockerfile.append("CMD %s\n" % cmd)
if version:
self.tag = "%s:%s" % (main_modul, self.get_modul_version(main_modul))
def add_betriebsystem_clean(self, base):
text = "RUN"
for i in range(1, 10):
if config.has_option("betriebsystem", "%s_%i" % (base, i)):
text = "%s %s \ \n &&" % (text, config.get("betriebsystem", "%s_%i" % (base, i)))
self.line_add(text[:-7])
def add_healthcheck(self, command, interval=config.getint("variable", "healthckeck_interval"),
timeout=config.getint("variable", "healthcheck_timeout"), httpbase="NONE"):
"""Fuegt eine HEALTHCHECK Zeile in das Dockerfile ein.\n\
Parameter:\n\
command: Testbefehl oder URL der Testseite\n\
interval: in Sekunden\n\
timeout: in Sekunden\n\
http: Flack ob das das comando ein Befehl ist oder eine URL"""
command = self.http_command(httpbase, command)
self.line_add("HEALTHCHECK --interval=%is --timeout=%is CMD %s || exit 1" % (interval, timeout, command))
# Dockerfile abarbeiten Methoden
def start_dockerfile(self, image="scratch", fromreg=True):
"""generiere Dockerfile Header\n\
image: Tag von Docker - Images das in FROM stehen soll oder welches Base Betribsystem eingebaut werden soll\n\
base: Betriebsystem auf den das Image basiert"""
image_list = image.split(":")
if fromreg and not image == "scratch":
image = "%s/%s" % (self.registry, image)
image_list[0] = "%s/%s" % (self.registry, image_list[0])
if not image == "scratch":
self.docker_deamon.pull(image_list[0], tag=image_list[1])
self.dockerfile.append("FROM %s\n" % image)
chdir(self.workdir)
def write_dockerfile(self, nogpg=False):
"""Schreibt das generirte Dockerfile ins Arbeitsverzeichnis\n\
Parameter: Flack um gpg abfragen in Orginal Dockerfile nicht zu uebernemen"""
fh = open(pjoin(self.dockerdir, self.dockername), "w")
for line in self.dockerfile:
if not nogpg:
fh.writelines(line)
else:
if line.find(".asc") > -1 or line.find(" gpg ") > -1:
pass
else:
fh.writelines(line)
def build_image(self, tag=None, nocache=False):
"""Generirt das Docker Images (Tagt die alte Version auf old um)\n\
Parameter: \n\
Tag: Tag von Image\n\
buildoption: Zusatz Optionen zum Build\n\
errordirclean: Flack ob nach einen fehlgeschlagenen Build das Arbeitsverzeichnis gesaubert werden soll oder nicht"""
self.tag = tag or self.tag
if self.tag is None:
stdout.write("\n")
stdout.write("Docker-Image Bezeichnung: \n\n")
self.tag = stdin.readline()[:-1]
stdout.write("\n")
if self.existst_tag(self.tag):
self.retag_and_push(self.tag, "%s:%s" % (self.tag_to_rep(), self.ttag), False)
output = [line for line in self.docker_deamon.build(self.dockerdir, self.tag, dockerfile=self.dockername, rm=True, nocache=nocache)]
for line in output:
line = eval(line)
if "stream" in line.keys():
print(line["stream"][:-1])
if "error" in line.keys():
print(line["error"])
exit(1)
if not self.existst_tag("%s:%s" % (self.tag_to_rep(), self.ttag), self.tag, True):
self.retag_and_push("%s:%s" % (self.tag_to_rep(), self.ttag), "%s:%s" % (self.tag_to_rep(), self.otag), False)
if self.existst_tag("%s:%s" % (self.tag_to_rep(), self.ttag)):
call(self.dockerrmi + ["%s:%s" % (self.tag_to_rep(), self.ttag)])
def test_image(self, cmd, waittime=config.getint("variable", "test_waittime"), noservice=False, toreg=True, httpbase=None):
"""Testet das generierte Image\n\
Parameter:\n\
waittime: Wartezeit bis der Test ausgefuert werden soll in Sekunden um denn Dienst Zeit zu geben zu starten (Default: 20s)\n\
cmd: Test Befehl\n\
toreg: Flack um zu bestimmen das das getestete Image in die Registry gepusht wird oder nicht (Default: True)\n\
noservice: Image ist kein Service-Image\n\
http: Flack ob das das comando ein Befehl ist oder eine URL"""
httpbase = httpbase or "NONE"
cmd = self.http_command(httpbase, cmd)
if noservice:
container = self.docker_deamon.create_container(image=self.tag, name="autotest_%s" % (self.tag.split(":")[0]), command="/bin/sh", tty=True)
else:
container = self.docker_deamon.create_container(image=self.tag, name="autotest_%s" % (self.tag.split(":")[0]), detach=True)
self.docker_deamon.start(container["Id"])
for i in tqdm(range(waittime), ascii=True, desc="Image Test in"):
sleep(1)
try:
execInstantz = self.docker_deamon.exec_create(container["Id"], cmd, tty=True)
except:
stdout.write("ERROR:\nImage ist abgestuerzt\n")
self.image_fail()
return False
out_text = self.docker_deamon.exec_start(execInstantz["Id"])
if self.docker_deamon.exec_inspect(execInstantz["Id"])["ExitCode"] > 0:
stdout.write("ERROR:\n%s\n" % out_text)
resolt = False
else:
resolt = True
self.docker_deamon.stop(container["Id"])
self.docker_deamon.remove_container(container["Id"])
if resolt:
self.to_registry(push=toreg)
else:
if not resolt:
self.image_fail()
exit(1)
# Registry Methoden
def to_registry(self, reg=None, push=True):
"""Bennent ein Docker-Image um fuer die Registry (Tagt die alte Version auf old um)\n\
Parameter: \n\
reg: Rgistry URL (Default: self.registry)\n\
push: Flack ob das Docker Image in das Registry gepusht werden soll"""
reg = reg or self.registry
if self.existst_tag("%s:%s" % (self.tag_to_rep(), self.otag)):
self.retag_and_push("%s:%s" % (self.tag_to_rep(), self.otag), "%s/%s:%s" % (reg, self.tag_to_rep(), self.otag), push)
self.retag_and_push(self.tag, "%s/%s" % (reg, self.tag), push)
self.retag_and_push(self.tag, "%s/%s:%s" % (reg, self.tag_to_rep(), self.ltag), push)
if self.production:
self.retag_and_push(self.tag, "%s/%s:%s" % (reg, self.tag_to_rep(), self.ptag), push)
def retag_and_push(self, oldtag, newtag, push=True):
"""Aendert ein Tag und Pusht das in die Registry\n\
Parameter: \n\
oldtag: existirendes Tag\n\
newtag: gewuenschtes Tag\n\
push: Flack ob das neu getackte Image ins Registry gepusht werden soll"""
call(self.dockertag + [oldtag, newtag])
if push:
call(self.dockerpush + [newtag])
# Docker Image sub Methoden
def existst_tag(self, tag, stag=None, test_id=False):
"""Sucht ob ein Tag existirt\n\
Parameter:\n\
tag: Tag der gesucht wird\n\
stag: Tag der mit tag verglichen wird wenn testID gesetzt ist\n\
test_id: Einzeltag suche (False) oder TagID vergleich von tag und stag (True)"""
stag = stag or ""
tag_id = "0"
sorce_id = "1"
image_list = self.docker_deamon.images(name="%s" % (self.tag_to_rep(tag)))
for image in image_list:
if image["RepoTags"] is None:
continue
imagetags = image["RepoTags"]
for imagetag in imagetags:
if imagetag == tag:
if not test_id:
return True
tag_id = image["Id"]
if imagetag == stag:
sorce_id = image["Id"]
if tag_id == sorce_id:
return True
return False
def image_fail(self):
call(self.dockertag + [self.tag, "%s:%s" % (self.tag_to_rep(), self.ftag)])
if self.existst_tag("%s:%s" % (self.tag_to_rep(), self.otag)):
self.retag_and_push("%s:%s" % (self.tag_to_rep(), self.otag), self.tag, False)
call(self.dockerrmi + ["%s:%s" % (self.tag_to_rep(), self.otag)])
# String umforatirung Methoden
@staticmethod
def http_command(httpbase, cmd):
if config.has_option("healthttp", httpbase):
return "%s %s" % (config.get("healthttp", httpbase), cmd)
else:
return cmd
@staticmethod
def path2file(pathlist):
"""Methode um aus einer Liste von Path Elementen ein Path-string zu erstellen\n\
Parameter: Liste aus Path Elementen"""
file = ""
for item in pathlist:
file = pjoin(file, item)
return file
def tag_to_rep(self, tag=None):
"""Gibt das Repository von einen Tag zurueck\n\
Parameter: ein Tag (Default: self.tag)"""
tag = tag or self.tag
split_tag = tag.split(":")
return ":".join(split_tag[:-1])
@staticmethod
def rawmodul_to_modul(rawmodul):
rawmodul = rawmodul.split(":", 1)
if len(rawmodul) > 1:
rawmodul = rawmodul[1]
else:
rawmodul = rawmodul[0]
rawmodul = rawmodul.split("#", 1)[0]
rawmodul = rawmodul.split(".", 1)[0]
return rawmodul
def get_modul_version(self, modul):
out = check_output(self.pipsearch + [self.pippurl] + [modul]).decode('utf-8')
out = out.split("(", 1)[1]
out = out.split(")", 1)[0]
return out
def snapshot_load(self, snapshots=(), volumes=()):
options = list(set(self.snapshot_ls()) & set(snapshots))
client = APIClient()
try:
client.inspect_image('iamdork/rsync')
except APIError:
client.pull('iamdork/rsync')
if len(options):
name = options[-1]
snapshot = '%s/%s' % (self.snapshot, name)
if not os.path.isdir(snapshot):
log.error("Snapshot %s of project %s doesn't exist." % (name, self.project))
return
for v in volumes:
log.info("Restoring volume %s from %s/%s." % (v, snapshot, v))
sync = client.create_container(
image='iamdork/rsync',
volumes=['/destination', '/source'],
host_config=client.create_host_config(binds=[
'%s/%s:/source' % (snapshot, v),
'%s:/destination' % volumes[v].full_name
]),
)
try:
client.start(sync)
while client.inspect_container(sync)['State']['Running']:
time.sleep(0.5)
finally:
client.remove_container(sync)
return name
return None
def snapshot_save(self, snapshots=(), volumes=()):
client = APIClient()
for name in snapshots:
snapshot = '%s/%s' % (self.snapshot, name)
for v in volumes:
log.info("Saving volume %s to %s/%s." % (v, snapshot, v))
try:
client.inspect_image('iamdork/rsync')
except APIError:
client.pull('iamdork/rsync')
sync = client.create_container(
image='iamdork/rsync',
volumes=['/destination', '/source'],
cpu_shares=256,
host_config=client.create_host_config(binds=[
'%s/%s:/destination' % (snapshot, v),
'%s:/source' % volumes[v].full_name
]),
)
try:
client.start(sync)
while client.inspect_container(sync)['State']['Running']:
time.sleep(0.5)
finally:
client.remove_container(sync)
def snapshot_rm(self, snapshots=()):
client = APIClient()
try:
client.inspect_image('alpine:3.4')
except APIError:
client.pull('alpine:3.4')
for name in snapshots:
snapshot = '%s/%s' % (self.snapshot, name)
if not os.path.isdir(snapshot):
log.error("Snapshot %s of project %s doesn't exist." % (name, self.project))
continue
container = client.create_container(
command='rm -rf /snapshots/%s' % name,
image='alpine:3.4',
volumes=['/snapshots'],
host_config=client.create_host_config(binds=[
'%s:/snapshots' % self.snapshot,
]),
)
try:
client.start(container)
while client.inspect_container(container)['State']['Running']:
time.sleep(0.5)
finally:
client.remove_container(container)
yield name