class Docker(object):
def __init__(self,base_url = ''):
self.cli = Client(base_url=base_url)
self.containers = []
self.images = 'docker/whalesay'
def build(self):
self.cli.build(path='../../dockfiles/'+self.images,rm=True,tag=self.images)
def create(self):
container = self.cli.create_container(image=self.images + ':latest',
command='cowsay boo',detach=True,
name=(self.images + ": " + (len(self.containers)-1).__str__()))
self.containers.append(container)
return len(self.containers) - 1
def remove(self,num):
self.stop(num)
container = self.containers[num]
self.cli.remove_container(container=container)
self.containers.remove(num)
def start(self,num):
self.cli.start(container=self.containers[num])
def stop(self,num):
self.cli.stop(container=self.containers[num])
def get_image_id(self):
image_name = 'container-%d:v%d' % (self.id, self.version)
path = os.path.join(settings.DOCKER_ROOT, 'build', image_name)
# create a client to communicate with docker
client = Client(base_url='unix://var/run/docker.sock')
# check if already built
images = client.images(name=image_name)
if images:
print('docker image exists: %s with name %s' % (self.tag, image_name))
return images[0]['Id']
# build the docker file
print('building new docker image: %s with name %s' % (self.tag, image_name))
extract_zip(self.dockerfile_src, path)
log = list(client.build(path=path, rm=True, tag=image_name))
self.build_log = "".join([str(i)+': '+str(log[i]) for i in range(len(log))])
self.save()
images = client.images(name=image_name)
if images:
return images[0]['Id']
else:
raise LookupError('Docker image not found: "' + self.tag + '"')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--nocache', action='store_true', default=False)
parser.add_argument('--dockerfile', type=local)
parser.add_argument('--label', choices=IMAGES.keys())
arguments = parser.parse_args()
content = ''
client = Client(base_url='unix://var/run/docker.sock')
stream = client.build(
tag=IMAGES[arguments.label],
dockerfile=arguments.dockerfile.basename,
path=arguments.dockerfile.dirname,
pull=True,
decode=True,
forcerm=True,
nocache=arguments.nocache
)
for item in stream:
buff = item.get('stream', item.get('status', ''))
if not content or re.search('.+\[[. ]*$', content):
content += buff
if not re.search('.+\[[. ]*$', content):
print(content)
content = ''
class DockerLayer(object):
def __init__(self, name, tag):
self.__name__ = name
self.__bases__ = tuple([])
self.tag = tag
self.client = Client(base_url='unix://var/run/docker.sock')
try:
self.client.ping()
except ConnectionError as e:
# http://docker-py.readthedocs.org/en/latest/boot2docker/
kwargs = kwargs_from_env()
kwargs['tls'].assert_hostname = False
self.client = Client(**kwargs)
def setUp(self):
if self.client.ping() == u'OK':
self.start()
else:
raise RuntimeError('Docker is not available.\nMake sure you have Docker installed before running tests.\nVisit https://docker.com for installation instructions.')
def start(self):
for line in self.client.build(
path=os.path.abspath(os.path.join(DIR, '..')),
tag=self.tag, rm=True, forcerm=True):
sys.stdout.write(line)
def tearDown(self):
self.stop()
def stop(self):
self.client.close()
def main(token, commit):
if token != os.getenv("TOKEN"):
abort(401)
working_copy = "/repository/checkout"
if not os.path.isdir(working_copy):
os.mkdir(working_copy)
Git().clone(os.getenv("GIT_REPOSITORY_URL"), working_copy)
print "Refreshing git repository"
repo = Repo(working_copy)
origin = repo.remotes.origin
origin.pull();
repo.git.reset(commit, hard=True)
print "Building docker image"
docker = Client(base_url='unix://mount/run/docker.sock')
docker.login(username=os.getenv("DOCKER_REGISTRY_USERNAME"), password=os.getenv("DOCKER_REGISTRY_PASSWORD"), email=os.getenv("DOCKER_REGISTRY_EMAIL", "*****@*****.**"))
before_build = os.getenv('BEFORE_BUILD_CMD')
if before_build is not None:
check_call(before_build, shell=True)
for line in docker.build(path=working_copy, tag=os.getenv("IMAGE_NAME"), stream=True):
print line
print "Pushing docker image"
for line in docker.push(os.getenv("IMAGE_NAME"), stream=True):
print line
print "Listing docker image"
check_call(os.getenv('DEPLOY_CMD'), shell=True)
return "OK"
def custom_docker_build(self):
from io import BytesIO
from docker import Client
import json
import re
doc_client = Client(base_url='unix://var/run/docker.sock',
version='1.18')
df1 = open(df_name, "r").read()
f = BytesIO(df1.encode('utf-8'))
resp = [
line for line in doc_client.build(
fileobj=f, rm=False, tag=cont_name + "-1")
]
try:
parsed_lines = [json.loads(e).get('stream', '') for e in resp]
except ValueError:
# sometimes all the data is sent on a single line ????
#
# ValueError: Extra data: line 1 column 87 - line 1 column
# 33268 (char 86 - 33267)
line = resp[0]
# This ONLY works because every line is formatted as
# {"stream": STRING}
parsed_lines = [
json.loads(obj).get('stream', '')
for obj in re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line)
]
# -----
fmlogger.error(parsed_lines)
def __init__(self,url_folder_language): cli = Client(base_url='unix://var/run/docker.sock') response = cli.build(path=url_folder_language,rm=True,nocache=True) for i in response: j=i[30:] id_container=j[:12] self.id_container=id_container
def get_log():
logs = []
json_data = json.loads(request.get_data())
project = ProjectModel.query.filter_by(proname=json_data['data']).first()
if not project.is_build():
os.chdir(app.config['CODE_FOLDER'] + '/' + json_data['data'])
cli = Client(base_url=HOST)
for line in cli.build(path=os.getcwd(), stream=True, decode=True,
tag=str(REGISTRY + json_data['data'])):
send_log(line)
logs.append(line)
# 向私有仓库推送镜像, 没有log的打印
for line in cli.push(REGISTRY + json_data['data'], stream=True):
# 打印出上传的log
print line
assert line
redis.hset(project.id, project.verify, logs)
project.build = True
if 'Successfully built' in logs[-1]['stream']:
project.success = 1
else:
project.success = 2
db.session.add(project)
db.session.commit()
else:
lines = eval(redis.hget(project.id, project.verify))
for line in lines:
send_log(line)
return json.dumps({'msg': 'ok'})
class DockerClient(object):
def __init__(self):
self.client = Client(base_url='unix://var/run/docker.sock')
def build_images(self, dockerfile: str, tag: str):
with open(dockerfile) as file:
dkfile = BytesIO(file.read().encode('utf-8'))
response = [line for line in self.client.build(
fileobj=dkfile, rm=True, tag=tag)]
return response
def run_container(self, image, mem_limit=None, volume_binds: list = None, command=None):
container = self.client.create_container(image=image,
host_config=self.client.create_host_config(
binds=volume_binds, mem_limit=mem_limit), command=command)
self.client.start(container)
try:
self.client.wait(container, timeout=3)
except ReadTimeout:
print('time out')
self.client.stop(container)
self.client.remove_container(container)
def exec_container(self, container, cmd):
container_id = self.client.exec_create(container, cmd)['Id']
return self.client.exec_start(container_id)
class DockerTest(object):
def __init__(self):
self.cli = Client(base_url='unix://var/run/docker.sock')
def build(self, dockerfile):
dockerfile = Path(dockerfile)
tag = 'rf' + dockerfile.basename().replace('Dockerfile.', '')
dockerfile.copy('redfish-client/tests/Dockerfile')
response = [line for line in self.cli.build(
path='redfish-client/tests',
tag=tag,
rm=True)]
return(response)
def run(self, image, command):
container = self.cli.create_container(image=image,
command=command,
tty=True,
stdin_open=True)
self.cli.start(container=container.get('Id'))
self.cli.wait(container=container.get('Id'))
response = self.cli.logs(container=container.get('Id'),
stdout=True)
self.cli.remove_container(container=container.get('Id'))
return(response.decode('utf8'))
def build_docker_image(docker_file_dir, docker_file, repository, log_file):
"""
Build docker image from the given docker file
Args:
docker_file_dir (path): Docker file dir
docker_file (path): Docker file
repository (str): Repo name
log_file (path): Log file path
Returns:
auth_config_payload (dict): AWS auth config
"""
docker_client = Client(base_url='unix://var/run/docker.sock')
write_to_debug_log(log_file,
"Creating Docker image: %s ..." % str(repository))
info = docker_client.build(dockerfile=docker_file,
tag=repository,
path=docker_file_dir,
rm=True,
stream=True)
with open(log_file, 'a') as f:
for item in info:
f.write("%s %s\n" % (" " * 10, str(item)))
write_to_debug_log(
log_file,
"Docker image: %s has been created locally!!!" % str(repository))
return docker_client
class DockerLayer(object):
def __init__(self, name, tag):
self.__name__ = name
self.__bases__ = tuple([])
self.tag = tag
self.client = Client(base_url='unix://var/run/docker.sock')
try:
self.client.ping()
except ConnectionError as e:
# http://docker-py.readthedocs.org/en/latest/boot2docker/
kwargs = kwargs_from_env()
kwargs['tls'].assert_hostname = False
self.client = Client(**kwargs)
def setUp(self):
if self.client.ping() == u'OK':
self.start()
else:
raise RuntimeError(
'Docker is not available.\nMake sure you have Docker installed before running tests.\nVisit https://docker.com for installation instructions.'
)
def start(self):
for line in self.client.build(path=os.path.abspath(
os.path.join(DIR, '..')),
tag=self.tag,
rm=True,
forcerm=True):
sys.stdout.write(line)
def tearDown(self):
self.stop()
def stop(self):
self.client.close()
class DockerTest(object):
def __init__(self):
self.cli = Client(base_url='unix://var/run/docker.sock')
def build(self, dockerfile):
dockerfile = Path(dockerfile)
tag = 'rf-' + dockerfile.basename().replace('.dkf', '')
dockerfile.copy('redfish-client/tests/Dockerfile')
response = [line for line in self.cli.build(
path='redfish-client/tests',
tag=tag,
rm=True)]
return(response)
def run(self, image, command):
container = self.cli.create_container(image=image,
command=command,
tty=True,
stdin_open=True)
self.cli.start(container=container.get('Id'))
self.cli.wait(container=container.get('Id'))
response = self.cli.logs(container=container.get('Id'),
stdout=True)
self.cli.remove_container(container=container.get('Id'))
return(response.decode('utf8'))
class DockerOpers(UtilOpers):
def __init__(self):
self.client = Client(base_url='unix://var/run/docker.sock',
version='1.12')
@staticmethod
def get_path_by__type(tp):
return options.dockerfile_dir + "/%s" % tp
def build(self, path, tag='', noCache=False):
streams = self.client.build(tag=tag,
path=path,
nocache=noCache,
rm=True)
res = [line for line in streams]
return res
def push(self, repository, tag=''):
res = self.client.push(repository, tag)
return res
def remove(self, repository, tag=''):
image = '%s:%s' % (repository, 'latest' if tag == '' else tag)
res = self.client.remove_image(image)
return res
def _create_docker_image(dockerfilepath, repository, pacbot_installation):
docker_client = Client(base_url='unix://var/run/docker.sock')
for info in docker_client.build(path=dockerfilepath,
tag=repository,
rm=True,
stream=True):
_logs_display(info, pacbot_installation)
return docker_client
def bake_main(options):
"""Munge a dockerfile from a cfg
cake:
layers: []
"""
endpoint = os.environ.get("LAYERCAKE_API")
if endpoint:
options.layer_endpoint = endpoint
config = yaml.load(open(options.config))['cake']
df = dockerfile.Dockerfile(options.dockerfile)
if options.layer_endpoint:
df.add("ENV", "LAYERCAKE_API={}".format(options.layer_endpoint))
# In this mode we are adding run cmds for each
# layer in the cfg file (those may pull other layers)
# then we output a new docker file and docker build the
# new container.
last_run = df.last("RUN")
if not options.use_devel:
df.add("RUN",
['pip3', 'install', '--upgrade',
'layer_cake==%s' % VERSION],
at=last_run)
else:
# For devel we ask the container to pull git master
df.add("RUN", [
'pip3', 'install', '--upgrade',
"https://api.github.com/repos/bcsaller/layercake/tarball/master#layer_cake"
],
at=last_run)
for layer_name in config['layers']:
last_run = df.last("RUN")
df.add("RUN", ["cake", "layer", layer_name, "-d", LAYERS_HOME],
at=last_run)
# we might have an entrypoint
# or a command (or both)
if df.entrypoint:
df.entrypoint = ["disco"] + df.entrypoint['args']
log.debug("Using Dockerfile\n%s", str(df))
if not options.no_build:
client = DockerClient()
f = BytesIO(str(df).encode("utf-8"))
response = client.build(fileobj=f, tag="layercake/disco", decode=True)
for line in response:
if 'errorDetail' in line:
log.critical(line['errorDetail']['message'].strip())
elif 'stream' in line:
log.info(line['stream'].strip())
else:
return df
def build_docker_container():
# Connect to docker
docker = DockerClient(base_url='unix://var/run/docker.sock')
# Build the container
r = [line for line in docker.build(
forcerm=True,
tag="transmission",
path=settings.BASE_DIR+'/dockerbuild/',
decode=True
)]
return r[-1]['stream'].find('Successfully built')
def build(ctx, image_name='ghcr.io/Daplanet/terraform-localexecute:latest'):
"""
Builds base docker image and pushes to <image_name>
"""
try:
cli = Client(base_url="unix:///var/run/docker.sock")
cli.login(registry=image_name.split('/', 1)[0],
user="******",
password=getenv('GITHUB_TOKEN'))
cli.build(path='.', tag=image_name)
cli.push(image_name)
except Exception as err:
print(err)
sys.exit(1)
finally:
del (cli)
def build_baton_docker(docker_client: Client, baton_docker_build: BatonImage):
"""
Builds the baton Docker image.
:param docker_client: the Docker client
:param baton_docker_build: where the baton docker is built from
"""
logging.info("Building baton Docker image - if this is not cached, it will take a few minutes")
logging.debug("baton Docker build: %s" % baton_docker_build)
# Note: reading the lines in this ways enforces that Python blocks - required
for line in docker_client.build(tag=baton_docker_build.tag, path=baton_docker_build.path,
dockerfile=baton_docker_build.docker_file, buildargs=baton_docker_build.build_args):
logging.debug(line)
def _build_images_from_dockerfiles(self):
"""
Build Docker images for each local Dockerfile found in the package: self.local_docker_files
"""
if GK_STANDALONE_MODE:
return # do not build anything in standalone mode
dc = DockerClient()
LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files))
for k, v in self.local_docker_files.iteritems():
for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False):
LOG.debug("DOCKER BUILD: %s" % line)
LOG.info("Docker image created: %s" % k)
def build_docker_image(nocache=False):
docker_client = Client(base_url='unix://var/run/docker.sock')
path = os.getcwd() + '/docker/'
image = ImageFactory(docker_client=docker_client, path=path)
return docker_client.build(
path=image['path'],
tag=image['tag'],
dockerfile=image['dockerfile'],
pull=True,
decode=True,
forcerm=True,
nocache=nocache
)
def build_docker_image(nocache=False, pull=True):
version = os.environ.get('VERSION', 'sles12sp1')
flavor = os.environ.get('FLAVOR') or 'products'
docker_client = Client(base_url='unix://var/run/docker.sock')
return docker_client.build(
tag='registry.mgr.suse.de/toaster-{0}-{1}'.format(version, flavor),
dockerfile='Dockerfile.{0}.{1}'.format(version, flavor),
path=os.getcwd() + '/docker/',
pull=pull,
decode=True,
forcerm=True,
nocache=nocache)
def create_container(cname, user, giturl):
try:
error = False
errmsg = ''
cid = ''
host_port = ''
print('nodeJsBuild start')
print('file creation')
cli = Client(base_url='unix://var/run/docker.sock')
#tls_config = tls.TLSConfig(
# client_cert=('/Users/kasi-mac/.docker/machine/machines/default/cert.pem', '/Users/kasi-mac/.docker/machine/machines/default/key.pem'),
# ca_cert='/Users/kasi-mac/.docker/machine/machines/default/ca.pem', verify=True)
#cli = Client(base_url='tcp://192.168.99.100:2376', tls=tls_config)
print('cli creation')
response = [
line
for line in cli.build(path=giturl, rm=True, tag=user + '/' + cname)
]
print(response)
print('cli creation end')
container = cli.create_container(
image=user + '/' + cname,
name=cname,
host_config=cli.create_host_config(publish_all_ports=True))
print(container)
ccrres = json.loads(json.dumps(container))
if 'Id' not in ccrres:
error = True
errmsg = 'Container creation failed'
else:
print('1')
cid = ccrres['Id']
print(cid)
response = cli.start(container=container.get('Id'))
print(response)
if response != None:
error = True
errmsg = 'Container failed to start'
else:
error = False
info = cli.inspect_container(container)
print info
host_port = 'http://' + str(
info['NetworkSettings']['Ports']['8080/tcp'][0]
['HostIp']) + ':' + str(info['NetworkSettings']['Ports']
['8080/tcp'][0]['HostPort'])
print host_port
return error, errmsg, cid, host_port
except Exception as inst:
print inst.args
return True, 'Exception', '', ''
def bake_main(options):
"""Munge a dockerfile from a cfg
cake:
layers: []
"""
endpoint = os.environ.get("LAYERCAKE_API")
if endpoint:
options.layer_endpoint = endpoint
config = yaml.load(open(options.config))['cake']
df = dockerfile.Dockerfile(options.dockerfile)
if options.layer_endpoint:
df.add("ENV", "LAYERCAKE_API={}".format(options.layer_endpoint))
# In this mode we are adding run cmds for each
# layer in the cfg file (those may pull other layers)
# then we output a new docker file and docker build the
# new container.
last_run = df.last("RUN")
if not options.use_devel:
df.add("RUN", ['pip3', 'install', '--upgrade', 'layer_cake==%s' % VERSION], at=last_run)
else:
# For devel we ask the container to pull git master
df.add("RUN", ['pip3', 'install', '--upgrade',
"https://api.github.com/repos/bcsaller/layercake/tarball/master#layer_cake"],
at=last_run)
for layer_name in config['layers']:
last_run = df.last("RUN")
df.add("RUN", ["cake", "layer", layer_name,
"-d", LAYERS_HOME],
at=last_run)
# we might have an entrypoint
# or a command (or both)
if df.entrypoint:
df.entrypoint = ["disco"] + df.entrypoint['args']
log.debug("Using Dockerfile\n%s", str(df))
if not options.no_build:
client = DockerClient()
f = BytesIO(str(df).encode("utf-8"))
response = client.build(fileobj=f, tag="layercake/disco", decode=True)
for line in response:
if 'errorDetail' in line:
log.critical(line['errorDetail']['message'].strip())
elif 'stream' in line:
log.info(line['stream'].strip())
else:
return df
def build_project(self, project_id, tag, **kwargs):
DOCKER_HOST = os.environ['DOCKER_HOST']
DOCKER_HOST = DOCKER_HOST.replace('tcp', 'https')
DOCKER_CERT_PATH = os.environ['DOCKER_CERT_PATH']
cert_dir = os.environ['DOCKER_CERT_PATH'] + '/'
tls_config = TLSConfig(
client_cert=(cert_dir + 'cert.pem', cert_dir + 'key.pem'), verify=False
)
docker_client = Client(base_url=DOCKER_HOST, tls = tls_config)
build = self.model()
self.validate_tag(tag)
build.tag = tag
image_name = "soma.buildbuild.io:4000/" + Project.objects.get(id = project_id).name + "-" + tag
build.project = Project.objects.get_project(project_id)
Dockerfile = self.optimize_docker_text(project_id = project_id)
try:
Build.objects.get(project = build.project, is_active = True)
except ObjectDoesNotExist:
pass
else:
old_build = Build.objects.get(project = build.project, is_active = True)
docker_client.stop(container = old_build.container)
# build.response = "".join([json.loads(line)["stream"] for line in
build.response = ([ line for line in
# docker_client.build(fileobj=open("/Users/Korniste/Developments/abc/Dockerfile"), tag=tag) ])
docker_client.build(fileobj=Dockerfile, rm=True, tag=image_name) ])
container = docker_client.create_container(
image=image_name,
name= build.project.name+"_"+build.tag,
detach=True,
ports = [ 8080 ]
)
build.container = container.get('Id')
build.port = 10000 + build.project.id
try:
Build.objects.get(project = build.project, is_active = True)
except ObjectDoesNotExist:
pass
else:
old_build = Build.objects.get(project = build.project, is_active = True)
docker_client.start(container = old_build.container , port_bindings = { 8080: old_build.port })
build.save(using = self.db)
return build
def build(self):
from docker import Client
from io import BytesIO
dockerhost = os.environ.get("DOCKER_HOST", "tcp://127.0.0.1:2375")
dockerfile = super(DockerBuild, self).build()
dockerfile = BytesIO(dockerfile.encode('utf-8'))
cli = Client(dockerhost)
builder = cli.build(fileobj=dockerfile, rm=True, tag="bones")
for line in builder:
line = json.loads(line)
if len(line.keys()) == 1 and "stream" in line:
sys.stdout.write(line["stream"])
sys.stdout.flush()
else:
print(line)
def build(repo, rev, image, hostname, location, socket='/var/run/docker.sock'):
'''
Build a docker image and install a salt minion, checkout the pull request
an perform a function test against the new code.
:param repo
The repository you would like to test against
:param rev
The revision to check
:param socket
The docket socket to perform all communication accorss
'''
docker = '''
FROM {0}
RUN yum -y install git wget dmidecode pciutils
RUN wget -O - https://bootstrap.saltstack.com | sh -s -- -X
RUN salt-call --local network.mod_hostname {1}
RUN git init /tmp/repo
WORKDIR /tmp/repo
RUN git config remote.origin.url {2}
RUN git fetch --tags --progress {2} +refs/pull/*:refs/remotes/origin/pr/*
RUN git pull
RUN git checkout -f {3}
RUN ln -s /tmp/repo/{4} /srv/salt
RUN salt-call --local state.highstate
'''.format(image, hostname, repo, rev, location)
cli = Client(base_url='unix:/{0}'.format(socket))
res = [i for i in cli.build(
fileobj=BytesIO(docker.encode('utf-8')), rm=True, nocache=True
)]
ret = re.search('Failed:\s+([0-9]{1,5})', res[-4])
if ret:
print('\n'.join(res[-4].strip().split('\\n'))[:-2])
return int(ret.group(1))
else:
print(res)
print('FAILED!!')
return 1
def build_docker_image(nocache=False, pull=True):
docker_client = Client(base_url='unix://var/run/docker.sock')
return docker_client.build(
tag=os.environ['DOCKER_IMAGE'],
dockerfile=os.environ['DOCKER_FILE'],
path=os.getcwd() + '/docker/',
pull=pull,
decode=True,
forcerm=True,
# Allows to invalidate cache for certains steps in Dockerfile
# https://github.com/docker/docker/issues/22832
buildargs={
'CACHE_DATE': datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
},
nocache=nocache)
def build_project(self, project_id, tag, **kwargs):
# Darwin is OS X
if platform.system() == 'Darwin':
DOCKER_HOST = os.environ['DOCKER_HOST']
DOCKER_HOST = DOCKER_HOST.replace('tcp', 'https')
DOCKER_CERT_PATH = os.environ['DOCKER_CERT_PATH']
cert_dir = os.environ['DOCKER_CERT_PATH'] + '/'
tls_config = TLSConfig(
client_cert=(cert_dir + 'cert.pem', cert_dir + 'key.pem'), verify=False
)
docker_client = Client(base_url=DOCKER_HOST, tls = tls_config)
if platform.system() == 'Linux':
docker_client = Client(base_url='unix://var/run/docker.sock')
build = self.model()
self.validate_tag(tag)
build.tag = tag
image_name = "soma.buildbuild.io:4000/" + Project.objects.get(id = project_id).name + "-" + tag
build.project = Project.objects.get_project(project_id)
Dockerfile = self.optimize_docker_text(project_id = project_id)
try:
Build.objects.get(project = build.project, is_active = True)
except ObjectDoesNotExist:
pass
else:
old_build = Build.objects.get(project = build.project, is_active = True)
docker_client.stop(container = old_build.container)
# build.response = "".join([json.loads(line)["stream"] for line in
build.response = ([ line for line in
# docker_client.build(fileobj=open("/Users/Korniste/Developments/abc/Dockerfile"), tag=tag) ])
docker_client.build(fileobj=Dockerfile, rm=True, tag=image_name, nocache=True) ])
try:
Build.objects.get(project = build.project, is_active = True)
except ObjectDoesNotExist:
pass
else:
old_build = Build.objects.get(project = build.project, is_active = True)
docker_client.start(container = old_build.container , port_bindings = { 8080: old_build.port })
build.image_name = image_name
docker_client.push(repository=image_name, insecure_registry=True)
build.save(using = self.db)
return build
def create_image(name,deploy_settings):
cli = Client(base_url='unix://var/run/docker.sock')
docker_path = os.path.join(deploy_settings['base_dir'], 'demokratikollen/deployment/docker/' + name)
full_name = 'demokratikollen/' + name
deploy_settings['log'].info("Creating image: {0}".format(name))
for line in cli.build(path=docker_path, rm=True, tag=full_name):
log_data = misc_utils.decode_docker_log(line)
if "error" in log_data:
deploy_settings['log'].error("Something went wrong with docker: {0}".format(log_data['error']))
raise Exception
else:
if 'stream' in log_data:
msg = log_data['stream'].strip()
if 'status' in log_data:
msg = log_data['status'].strip()
deploy_settings['log'].debug("Docker: {0}".format(msg))
def build_docker_image(docker_file_dir, docker_file, repository, log_file):
docker_client = Client(base_url='unix://var/run/docker.sock')
write_to_debug_log(log_file,
"Creating Docker image: %s ..." % str(repository))
info = docker_client.build(dockerfile=docker_file,
tag=repository,
path=docker_file_dir,
rm=True,
stream=True)
with open(log_file, 'a') as f:
for item in info:
f.write("%s %s\n" % (" " * 10, str(item)))
write_to_debug_log(
log_file,
"Docker image: %s has been created locally!!!" % str(repository))
return docker_client
def dock_image(dockerfile_path):
"""generate docker image"""
status = True
dc = DockerClient(base_url=CONF.clients_docker.url)
dc.info()
with open(dockerfile_path) as dockerfile:
tag = re.findall("(?im)^# tag: (.*)$", dockerfile.read())[0].strip()
LOG.debug("Generating %s from %s" % (tag, dockerfile_path))
if tag:
resp = dc.build(path=os.path.dirname(dockerfile_path),
dockerfile=os.path.basename(dockerfile_path),
rm=True,
tag=tag)
for l in resp:
if "error" in l.lower():
status = False
LOG.debug(l.strip())
return status
def dock_image(df):
"""generate docker image"""
status = True
dc = DockerClient(base_url=CONF.clients_docker.url)
dc.info()
with open(df) as dockerfile:
tag = re.findall("(?im)^# tag: (.*)$", dockerfile.read())[0].strip()
LOG.debug("Generating %s from %s" % (tag, df))
resp = dc.build(
fileobj=dockerfile,
rm=True,
tag=tag
)
for l in resp:
if "error" in l.lower():
status = False
LOG.debug(l)
return status
def docker_build( build_dir, tag="cyledge/base", dockerfile="Dockerfile", pull_first=True, use_cache=True, quiet=False):
logger = logging.getLogger("docker")
logging.info("Building docker image %s" % tag)
if pull_first:
logging.debug("Pulling FROM image first")
if not use_cache:
logging.debug("Not using cache")
#dockerfile = "%s/%s" % (build_dir, dockerfile)
c = Client(base_url='unix://var/run/docker.sock')
build_output = c.build(
path=build_dir,
dockerfile=dockerfile,
tag=tag,
stream=True,
quiet=quiet,
pull=pull_first,
nocache= not use_cache
)
last_line = None
for line in build_output:
output = json.loads(line.decode('UTF-8'))
if "stream" in output:
logger.debug(output["stream"].rstrip())
last_line = output["stream"]
if "error" in output:
logger.error(output["error"].rstrip())
#logger.error(output["errorDetail"])
if quiet:
srch = r'sha256:([0-9a-f]{12})[0-9a-f]+'
else:
srch = r'Successfully built ([0-9a-f]{12})'
match = re.search(srch, last_line)
if not match:
raise RuntimeError()
else:
return match.group(1)
def build(self, path, tag):
"""
Similar to the `docker interpreter`, interpreter a docker image
:param path: context path include Dockerfile
:param tag: image's tag
:return: None
"""
self.read_port()
version = self.app['docker']['api']['version']
cli = Client(base_url=self.url, version=str(version))
response = cli.build(path, tag, rm=True)
for line in response:
rp = {
key: str(item.strip().decode('unicode_escape'))
for key, item in ast.literal_eval(line).items()
}
log.info(rp)
log.info("successful build image with dockerImageTag=%s",
str(tag).split(':')[1])
def build_image(git_auth,
image_name,
git_repo,
image_tag='latest',
git_branch=None,
git_directory=None):
cli = Client(base_url='unix://var/run/docker.sock',
version=docker_api_version)
if not git_repo.endswith('.git'):
logger.debug('Adding .git to {}'.format(git_repo))
git_repo += '.git'
git_repo = 'https://' + git_auth + '@' + git_repo[8:]
if git_directory:
logger.debug('Adding : to {}'.format(git_directory))
git_directory = ':' + git_directory
if git_branch is None:
logger.debug('Adding # to {}'.format(git_branch))
git_branch = '#'
else:
logger.debug('Adding # to {}'.format(git_branch))
git_branch = '#' + git_branch
else:
git_directory = ''
if not git_branch.startswith('#'):
logger.debug('Adding # to {}'.format(git_branch))
git_branch = '#' + git_branch
logger.debug(git_repo + git_branch + git_directory)
logger.debug(image_name + ':' + image_tag)
try:
response = [
line for line in cli.build(git_repo + git_branch + git_directory,
rm=True,
tag=image_name + ':' + image_tag)
]
except docker_error as msg:
logger.error('Something went wrong:\n{}'.format(msg))
return msg
else:
logger.debug(response)
return None
def bootstrap(_: Namespace) -> None:
"""Create the images used by son-analyze in the current host"""
cli = Client(base_url='unix://var/run/docker.sock')
root_context = os.path.realpath(
resource_filename('son_analyze.cli', '../../..'))
_LOGGER.info('The root context path is: %s', root_context)
path = resource_filename('son_analyze.cli.resources',
'anaconda.Dockerfile')
path = os.path.relpath(path, root_context)
_LOGGER.info('The relative path to the bootstrap dockerfile is: %s', path)
# import pdb; pdb.set_trace()
for line in cli.build(path=root_context, tag=_IMAGE_TAG,
dockerfile=path, rm=True, decode=True):
if "stream" in line:
print('> ', line["stream"], end="")
else:
print(line)
sys.exit(1)
sys.exit(0)
def build_image( df, **options ):
debug = False
test = False
input_data = df
if 'debug' in options and options['debug'] in [True, False]: debug = options['debug']
if 'test' in options and options['test'] in [True, False]: test = options['test']
cli = Client( )
try:
f = BytesIO( df.as_string().encode('utf-8'))
response = [line for line in cli.build( fileobj=f, rm=True, tag=df.get_tag() )]
if debug:
print("=========================================")
pprint( response )
print("=========================================")
xty = eval( response[-1].decode() )
if 'stream' in xty and re.match( r"^Successfully", xty['stream'] ):
print("Completed build")
return True
elif 'errorDetail' in xty:
print("Error in build: %(err)s" % {'err': xty['errorDetail']['message'] })
return False
else:
print("Unexpected result: ")
pprint( xty )
return False
except Exception as error:
pprint( error )
return False
return True
def _build_image_on_docker_host(self, base_url, build_file, dockerfile,
image_name, image_version):
"""
Build image on the selected docker host by Dockerfile.
'base_url': the url of docker host.
'build_file': the name of the build file in absolute path.
'dockerfile': Dockerfile path in build_file.
'image_name': the name of the image, containing registry address, user
name and image name.
'image_version': the version of the image.
Returns:
'token': the image token
"""
self._delete_image_on_docker_host(base_url, image_name, image_version)
client = Client(base_url=base_url)
fileobj = open(build_file, 'rb')
image_complete_name = '%s:%s' % (image_name, image_version)
try:
response = [
line for line in client.build(fileobj=fileobj,
custom_context=True,
dockerfile=dockerfile,
rm=True,
tag=image_complete_name)
]
except APIError as error:
logger.debug(error)
logger.error('Cannot locate specified Dockerfile: %s.' %
(self.dockerfile))
fileobj.close()
return None
except Exception as error:
logger.debug(error)
logger.error('Build image %s failed.' % image_complete_name)
fileobj.close()
return None
fileobj.close()
return self._get_image_token(base_url, image_complete_name)
def _build_image_on_docker_host(self, base_url, build_file, dockerfile,
image_name, image_version):
"""
Build image on the selected docker host by Dockerfile.
'base_url': the url of docker host.
'build_file': the name of the build file in absolute path.
'dockerfile': Dockerfile path in build_file.
'image_name': the name of the image, containing registry address, user
name and image name.
'image_version': the version of the image.
Returns:
'token': the image token
"""
self._delete_image_on_docker_host(base_url, image_name, image_version)
client = Client(base_url=base_url)
fileobj = open(build_file, 'rb')
image_complete_name = '%s:%s' % (image_name, image_version)
try:
response = [line for line in client.build(
fileobj=fileobj,
custom_context=True,
dockerfile=dockerfile,
rm=True,
tag=image_complete_name)]
except APIError as error:
logger.debug(error)
logger.error('Cannot locate specified Dockerfile: %s.' %
(self.dockerfile))
fileobj.close()
return None
except Exception as error:
logger.debug(error)
logger.error('Build image %s failed.' % image_complete_name)
fileobj.close()
return None
fileobj.close()
return self._get_image_token(base_url, image_complete_name)
def build_image(client: docker.Client, path='./remote-docker', name='noodles-remote'):
"""Build the Docker image as per Dockerfile present in the 'remote-docker'
sub-directory. This Docker image should suffice to emulate a remote machine
with SLURM queue manager and ssh/sftp connection installed. If the docker image
with given name is newer than the Dockerfile, nothing is done.
:param client:
Docker client
:param path:
Location of Dockerfile
:param name:
Name of the image
"""
assert os.path.exists(path + '/Dockerfile')
time = os.stat(path + '/Dockerfile').st_mtime
il = client.images(name=name)
if len(il) == 0 or il[0]['Created'] < time:
response = client.build(path, tag=name, rm=True)
for json_bytes in response:
line = json.loads(json_bytes.decode())['stream']
print(line, end='', file=sys.stderr, flush=True)
def dock_image():
"""generate docker image"""
import sys
import os
status = True
dc = DockerClient(base_url=CONF.clients_docker.url)
# inject config files dir to syspath
print CONF.config_dir
wp = os.path.abspath(CONF.config_dir)
sys.path.insert(0, wp)
os.chdir(wp)
with open("ChefImage.docker") as dockerfile:
resp = dc.build(
fileobj=dockerfile,
rm=True,
tag=CONF.clients_docker.image
)
for l in resp:
if "error" in l.lower():
status = False
LOG.debug(l)
return status
def build_docker_image(request):
assert isinstance(request,HttpRequest)
# filter other type of request
if request.method != "POST" and request.POST:
return HttpResponseBadRequest()
# dummy check just to supress the error when
# accesing POST directly, that is Django's pitfall
if len(request.POST) == 0:
pass
str_json = ""
try:
str_json = request.body.decode("utf-8")
except Exception as e:
return HttpResponseBadRequest(str(e))
indata = json.loads(str_json)
indata['user_id'] = request.user.id
get_resp = json.loads(GetDockerImageLogic().execute(indata))
docker_image = get_resp['data']
try:
cl = Client(settings.DOCKER_CLIENT)
f = BytesIO(docker_image['content'].encode('utf-8'))
if settings.DOCKER_SINGLE_HOST == True:
image_tag = "%s/%s/%s:%s" % ("127.0.0.1:5000",docker_image['user'],docker_image['name'],docker_image['version'])
else:
image_tag = "%s/%s/%s:%s" % (settings.DOCKER_REGISTRY,docker_image['user'],docker_image['name'],docker_image['version'])
return StreamingHttpResponse(output for output in cl.build(fileobj=f, rm=True, tag=image_tag))
except Exception as e:
res = LogicReturn(LogicStatus.FAILED,str(e))
return HttpResponse(res.to_JSON(), content_type="application/json")
def build(self):
"""
runs docker build with the tarfile context
"""
docker = Client(version='auto')
status = docker.build(
fileobj=self.tarfile,
custom_context=True,
tag=self.tag,
pull=True,
nocache=True,
rm=True,
)
for line in status: # This effectively blocks on `docker build`
try:
current_app.logger.debug(line)
except RuntimeError: # Outside of application context
print line
if "successfully built" not in line.lower():
raise BuildError("Failed to build {}: {}".format(self.tag, line))
self.built = True
def build(self):
"""
runs docker build with the tarfile context
"""
docker = Client(version='auto')
status = docker.build(
fileobj=self.tarfile,
custom_context=True,
tag=self.tag,
pull=True,
nocache=True,
rm=True,
)
for line in status: # This effectively blocks on `docker build`
try:
current_app.logger.debug(line)
except RuntimeError: # Outside of application context
print line
if "successfully built" not in line.lower():
raise BuildError("Failed to build {}: {}".format(self.tag, line))
self.built = True
def build_image(client: docker.Client,
path='./remote-docker',
name='noodles-remote'):
"""Build the Docker image as per Dockerfile present in the 'remote-docker'
sub-directory. This Docker image should suffice to emulate a remote machine
with SLURM queue manager and ssh/sftp connection installed. If the docker image
with given name is newer than the Dockerfile, nothing is done.
:param client:
Docker client
:param path:
Location of Dockerfile
:param name:
Name of the image
"""
assert os.path.exists(path + '/Dockerfile')
time = os.stat(path + '/Dockerfile').st_mtime
il = client.images(name=name)
if len(il) == 0 or il[0]['Created'] < time:
response = client.build(path, tag=name, rm=True)
for json_bytes in response:
line = json.loads(json_bytes.decode())['stream']
print(line, end='', file=sys.stderr, flush=True)
class DockerHelper(object):
def __init__(self, logger=None, base_url=""):
self.logger = logger or create_file_logger()
self.docker = Client(base_url=base_url or "unix:///var/run/docker.sock")
def image_exists(self, name):
"""Checks whether a docker image exists.
:param name: Image name
:returns: ``True`` if image exists, otherwise ``False``
"""
images = self.docker.images(name)
return True if images else False
def build_image(self, path, tag):
"""Builds a docker image.
:param path: Path to a directory where ``Dockerfile`` is located.
:param tag: Desired tag name.
:returns: ``True`` if image successfully built, otherwise ``False``
"""
self.logger.info("building {} image".format(tag))
resp = self.docker.build(path, tag=tag, quiet=True,
rm=True, forcerm=True)
output = ""
while True:
try:
output = resp.next()
self.logger.info(output)
except StopIteration:
break
result = json.loads(output)
if "errorDetail" in result:
return False
return True
def run_container(self, name, image):
"""Runs a docker container in detached mode.
This is a two-steps operation:
1. Creates container
2. Starts container
:param name: Desired container name.
:param image: Existing image name.
:returns: A string of container ID in long format if container
is running successfully, otherwise an empty string.
"""
container_id = ""
self.logger.info("creating container {!r}".format(name))
env = {
"SALT_MASTER_IPADDR": os.environ.get("SALT_MASTER_IPADDR"),
}
container = self.docker.create_container(
image=image, name=name, detach=True, environment=env,
)
container_id = container["Id"]
self.logger.info("container {!r} has been created".format(name))
if container_id:
self.docker.start(container=container_id)
self.logger.info("container {!r} with ID {!r} "
"has been started".format(name, container_id))
return container_id
def get_remote_files(self, *files):
"""Retrieves files from remote paths.
All retrieved files will be stored under a same temporary directory.
:param files: List of files.
:returns: Absolute path to temporary directory where all files
were downloaded to.
"""
local_dir = tempfile.mkdtemp()
for file_ in files:
local_path = os.path.join(local_dir, os.path.basename(file_))
self.logger.info("downloading {!r}".format(file_))
resp = requests.get(file_)
if resp.status_code == 200:
with open(local_path, "w") as fp:
fp.write(resp.text)
return local_dir
def _build_gluubase(self):
"""Builds gluubase image.
:param salt_master_ipaddr: IP address of salt-master.
:returns: ``True`` if image successfully built, otherwise ``False``
"""
build_succeed = True
if not self.image_exists("gluubase"):
# There must be a better way than to hard code every file one by one
DOCKER_REPO = 'https://raw.githubusercontent.com/GluuFederation' \
'/gluu-docker/master/ubuntu/14.04'
minion_file = DOCKER_REPO + '/gluubase/minion'
supervisor_conf = DOCKER_REPO + '/gluubase/supervisord.conf'
render = DOCKER_REPO + '/gluubase/render.sh'
dockerfile = DOCKER_REPO + '/gluubase/Dockerfile'
files = [minion_file, supervisor_conf, render, dockerfile]
build_dir = self.get_remote_files(*files)
build_succeed = self.build_image(build_dir, "gluubase")
shutil.rmtree(build_dir)
return build_succeed
def setup_container(self, name, image, dockerfile, salt_master_ipaddr):
"""Builds and runs a container.
:param name: Container name.
:param image: Image name.
:param dockerfile: Path to remote Dockerfile. Used to build the image
if image is not exist.
:returns: Container ID in long format if container running successfully,
otherwise an empty string.
"""
if not self._build_gluubase():
return ""
# a flag to determine whether build image process is succeed
build_succeed = True
if not self.image_exists(image):
build_dir = self.get_remote_files(dockerfile)
build_succeed = self.build_image(build_dir, image)
shutil.rmtree(build_dir)
if build_succeed:
return self.run_container(name, image)
return ""
def get_container_ip(self, container_id):
"""Gets container IP.
:param container_id: Container ID; ideally the short format.
:returns: Container's IP address.
"""
info = self.docker.inspect_container(container_id)
return info["NetworkSettings"]["IPAddress"]
def remove_container(self, container_id):
"""Removes container.
"""
try:
return self.docker.remove_container(container_id, force=True)
except docker.errors.APIError as exc:
err_code = exc.response.status_code
if err_code == 404:
self.logger.warn(
"container {!r} does not exist".format(container_id))
def set_container():
global container_id
global cli
global render_path
global config_path
global watch_path
log = logging.getLogger(__name__)
### Connecting to Docker Client API
if os.name == 'nt':
# Windows
cli = Client(base_url='tcp://127.0.0.1:2375')
else:
# Unix
cli = Client(base_url='unix://var/run/docker.sock')
### Just one instance running at a time
running = cli.containers(all=True, filters={"name": CONTAINER_NAME})
log.info("{} containers in host: {}".format(CONTAINER_NAME, len(running)))
if len(running) > 0:
container_id = running[0].get('Id')
log.info("Stopping or removing old container {}".format(container_id))
#response = cli.kill(container=container_id, signal=signal.SIGTERM)
#print("EXITED")
#exit(1)
cli.stop(container=container_id)
cli.remove_container(container=container_id)
### Build docker image from Dockerfile
log.info("Building container...")
try:
response = [
line for line in cli.build(
path='./image', tag=IMAGE_NAME, rm=True, forcerm=True)
]
except Exception as error:
log.error("ERROR: Missing the Dockerfile to build the container %s" %
error)
exit(1)
for line in response:
if "error" in line:
log.error("ERROR: Missing /app folder inside the image.")
exit(2)
log.info("Container built succesfully.")
### Creating render structure for mounting on /app/render
uri = [
"AppData", "Roaming", "com.dedosmedia.SkinRetouching", "Local Store",
"content", "Keshot", "compositions", "master", "input"
]
render_path = os.path.join(os.path.expanduser("~"), *uri)
if not os.path.exists(render_path):
log.info("Creating rendering path on local disk.")
try:
os.makedirs(render_path)
except OSError as error:
log.error(
"ERROR: creating rendering structure on local disk. {}".format(
error))
exit(1)
render_path = os.path.realpath(os.path.dirname(render_path))
#print("Render path is: {} and exists: {}".format(render_path, os.path.exists(render_path)))
### Config path to mount on /app/config
config_path = os.path.realpath(os.path.dirname(CONFIG_FILENAME))
log.info("Config path is: {} and exists: {}".format(
config_path, os.path.exists(config_path)))
### Watch path to mount on /app/watch
watch_path = os.path.realpath(config["watch-folder-" + os.name])
log.info("Watch path is: {} and exists: {}".format(
watch_path, os.path.exists(watch_path)))
try:
folder = os.path.realpath(
os.path.join(watch_path, config['output-subfolder']))
os.makedirs(folder)
except OSError as error:
if os.path.exists(folder) == False:
log.error(
"ERROR: creating watch output structure on local disk. {}".
format(error))
exit(1)
### Setting up volumes and port bindings
host_config = cli.create_host_config(
binds={
config_path: {
'bind': posixpath.normpath(
posixpath.join(APP_PATH, CONFIG_PATH)), #'/app/config',
'mode': 'rw'
},
render_path: {
'bind': posixpath.normpath(
posixpath.join(APP_PATH, RENDER_PATH)), #'/app/render',
'mode': 'rw'
},
watch_path: {
'bind':
posixpath.normpath(posixpath.join(APP_PATH,
WATCH_PATH)), #'/app/watch',
'mode': 'rw'
}
},
port_bindings={PORT: PORT})
### Creating and starting the container
container = cli.create_container(
image=IMAGE_NAME,
#volumes= ['/app/config','/app/render','/app/watch'],
#ports=[4999],
detach=True,
environment={
"CONFIG": CONFIG_FILENAME,
"PORT": PORT,
"CORE": CORE_FILENAME,
"WATCH": WATCH_PATH,
"RENDER": RENDER_PATH,
"HOST_RENDER_PATH": render_path,
"OUTPUT_NAME": OUTPUT_NAME
},
host_config=host_config,
name=CONTAINER_NAME)
log.info("Container created: {}".format(container))
container_id = container.get('Id')
response = cli.start(container=container_id)
log.info("Container started correctly. Monitoring: {}".format(
os.path.realpath(os.path.join(watch_path, config["input-file"]))))
#Get the port the docker host is listening on
port = get_docker_port(docker)
######## Connect to the docker host ########
#Connect to the remote docker host in the LXC container
dockerclient = Dclient(base_url="tcp://{}:{}".format(host,port), version="auto")
print "Connected to Docker host on {}:{}".format(host,port)
######## Create an docker image ########
print "\nCreating image from Dockerfile\n"
#Create a docker container image from the Dockerfile in this directory
response = dockerclient.build(path=".", dockerfile="Dockerfile", tag="pyne/helloworld")
#Show the output from the build process
for line in response:
print line
#Show the images now avalible on the docker host
print "Images on docker host:"
pprint(dockerclient.images())
print "\nCreating container from hello world image:\n"
#Create a config object containing the port mappings from the containers port to the host port
#The container port should match the one that is exposed in the Dockerfile
container_port = 5000
host_port = 4000
#Get the hostname
hostname = socket.gethostname()
imagename = hostname+'/'+app
#Connect to docker socket
cli = Client(base_url='unix://docker.sock')
#Define port binding
if dockerized:
config=cli.create_host_config(network_mode='container:'+yunohostid)
else:
config=cli.create_host_config(port_bindings={8000: ('127.0.0.1',8000)})
#Build docker image with the Dockerfile and disply the output
for line in cli.build(path='../build/', tag=imagename):
out=json.loads(line)
#sys.stdout.write('\r')
#print(out['stream'])
#sys.stdout.flush()
#Create the container and display result
container = cli.create_container(
image=imagename,
detach=True,
tty=True,
name=containername,
host_config=config
)
#Start the container and display result
#!/usr/bin/python #Se importa el cliente docker. from docker import Client from io import BytesIO dockerfile = ''' # Shared Volume FROM busybox MAINTAINER Ernesto Crespo, [email protected] VOLUME /data CMD ["/bin/sh"] ''' f = BytesIO(dockerfile.encode('utf-8')) cli = Client(base_url='unix://var/run/docker.sock') response = [line for line in cli.build( fileobj=f, rm=True, tag='yourname/volume' )] for i in response: print (i) print (cli.containers())
class Docker(object):
def __init__(self,
socket,
remove_intermediate=False,
registry=None,
insecure_registry=False):
log.debug('DockerClient connecting to {}'.format(socket))
self._socket = socket
self._client = Client(base_url=socket, timeout=300)
self._remove_intermediate = remove_intermediate
self._registry = registry
self._insecure_registry = insecure_registry
@exception_safe(ConnectionError, False)
def build(self, dockerfile, image):
"""/
:param dockerfile: The dockerfile full path
:param image: the image name (eg: midolman:1.9)
"""
log.info('Now building {}'.format(image))
log.debug('Invoking docker build on {}'.format(dockerfile))
response = self._client.build(path=os.path.dirname(dockerfile),
tag=image,
pull=False,
rm=self._remove_intermediate,
dockerfile=os.path.basename(dockerfile))
last_line = None
for line in response:
eval_line = eval(line)
if 'stream' in eval_line:
print(eval_line['stream']),
last_line = eval_line['stream']
# Check for ensure success after building
if 'Successfully built' not in last_line:
log.error('Error building the image {}.'.format(image))
return False
return True
@exception_safe(ConnectionError, None)
def pull(self, image):
"""
:param image: The image name with its tag
:param repository: The repository where to pull the image
(dockerhub if none defined)
"""
name, tag = image.split(':')
repository = '{}/{}'.format(self._registry, name) \
if self._registry else name
log.info('Now Pulling {}:{}'.format(repository, tag))
response = self._client.pull(repository=repository,
tag=tag,
insecure_registry=self._insecure_registry,
stream=True)
for line in response:
eval_line = eval(line)
if 'error' in eval_line:
log.error('Error pulling image: {}'.format(eval_line['error']))
return False
if 'status' in eval_line:
log.info('[{}:{}] Status: {}'.format(repository, tag,
eval_line['status']))
if self._registry:
# If pulling from an external repo, we need to tag with the
# actual image name used in the flavours definition.
images = self.list_images(repository)
for image in images:
for repotag in image['RepoTags']:
if '{}:{}'.format(repository, tag) == repotag:
self._client.tag(image['Id'], name, tag, force=True)
return True
@exception_safe(ConnectionError, None)
def push(self, image):
"""
:param image: The image name with its tag
:param repository: The repository where to push the image
(dockerhub if none defined)
"""
name, tag = image.split(':')
if self._registry:
# First tag the local image to map to the new registry
repository = '{}/{}'.format(self._registry, name)
images = self.list_images(name)
for image in images:
for repotag in image['RepoTags']:
if '{}:{}'.format(name, tag) == repotag:
self._client.tag(image['Id'],
repository,
tag,
force=True)
else:
repository = name
log.info('Now pushing {}:{}'.format(repository, tag))
response = self._client.push(repository=repository,
tag=tag,
insecure_registry=self._insecure_registry,
stream=True)
for line in response:
eval_line = eval(line)
if 'error' in eval_line:
log.error('Error pushing image: {}'.format(eval_line['error']))
return False
if 'status' in eval_line:
if 'Pushing' not in eval_line['status'] \
and 'Buffering' not in eval_line['status']:
log.info('[{}:{}] Status: {}'.format(
repository, tag, eval_line['status']))
return True
@exception_safe(ConnectionError, [])
def list_images(self, prefix=None):
"""
List the available images
:param prefix: Filter the images by a prefix (eg: "sandbox/")
:return: the images list
"""
images = self._client.images()
if prefix:
filtered = list()
for image in images:
for tag in image['RepoTags']:
if tag.startswith(prefix):
filtered.append(image)
images = filtered
return images
def list_containers(self, prefix=None):
"""
List the running containers, prefixed with prefix
:param prefix: The container's name prefix
:return: The list of containers
"""
containers = self._client.containers()
filtered = list()
if prefix:
for container_ref in containers:
if prefix in container_ref['Names'][0]:
filtered.append(container_ref)
containers = filtered
return containers
def container_by_name(self, name):
containers = self.list_containers()
for container_ref in containers:
if name == self.principal_container_name(container_ref):
return container_ref
@staticmethod
def principal_container_name(container_ref):
for name in container_ref['Names']:
if '/' not in name[1:]:
return name[1:]
def container_ip(self, container_ref):
return self._client.inspect_container(
container_ref)['NetworkSettings']['IPAddress']
def stop_container(self, container_ref):
self._client.stop(container_ref)
def kill_container(self, container_ref):
self._client.kill(container_ref)
def remove_container(self, container_ref):
self._client.remove_container(container_ref)
@exception_safe(OSError, False)
def execute(self, container_ref, command):
"""
Execute a command inside the container.
NOTE: Needs the 'docker' binary installed in the host
"""
cmd = [
'docker', 'exec', '-it',
self.principal_container_name(container_ref), 'env', 'TERM=xterm',
command
]
log.debug('Running command: "{}"'.format(' '.join(cmd)))
p = subprocess.Popen(cmd, stderr=subprocess.STDOUT)
p.wait()
def ssh(self, container_ref):
self.execute(container_ref, 'bash')
class TestRunner(object):
"""Badwolf test runner"""
def __init__(self, context, lock):
self.context = context
self.lock = lock
self.repo_full_name = context.repository
self.repo_name = context.repository.split('/')[-1]
self.task_id = str(uuid.uuid4())
self.commit_hash = context.source['commit']['hash']
self.build_status = BuildStatus(
bitbucket, context.source['repository']['full_name'],
self.commit_hash, 'badwolf/test',
url_for('log.build_log', sha=self.commit_hash, _external=True))
self.docker = Client(
base_url=current_app.config['DOCKER_HOST'],
timeout=current_app.config['DOCKER_API_TIMEOUT'],
)
def run(self):
start_time = time.time()
self.branch = self.context.source['branch']['name']
try:
self.clone_repository()
except git.GitCommandError as e:
logger.exception('Git command error')
self.update_build_status('FAILED', 'Git clone repository failed')
content = ':broken_heart: **Git error**: {}'.format(to_text(e))
if self.context.pr_id:
pr = PullRequest(bitbucket, self.repo_full_name)
pr.comment(self.context.pr_id, content)
else:
cs = Changesets(bitbucket, self.repo_full_name)
cs.comment(self.commit_hash, content)
self.cleanup()
return
if not self.validate_settings():
self.cleanup()
return
context = {
'context':
self.context,
'task_id':
self.task_id,
'build_log_url':
url_for('log.build_log', sha=self.commit_hash, _external=True),
'branch':
self.branch,
'scripts':
self.spec.scripts,
}
if self.spec.scripts:
self.update_build_status('INPROGRESS', 'Test in progress')
docker_image_name, build_output = self.get_docker_image()
context['build_logs'] = to_text(build_output)
context.update({
'build_logs': to_text(build_output),
'elapsed_time': int(time.time() - start_time),
})
if not docker_image_name:
self.update_build_status('FAILED',
'Build or get Docker image failed')
context['exit_code'] = -1
self.send_notifications(context)
self.cleanup()
return
exit_code, output = self.run_tests_in_container(docker_image_name)
if exit_code == 0:
# Success
logger.info('Test succeed for repo: %s', self.repo_full_name)
self.update_build_status('SUCCESSFUL', '1 of 1 test succeed')
else:
# Failed
logger.info('Test failed for repo: %s, exit code: %s',
self.repo_full_name, exit_code)
self.update_build_status('FAILED', '1 of 1 test failed')
context.update({
'logs': to_text(output),
'exit_code': exit_code,
'elapsed_time': int(time.time() - start_time),
})
self.send_notifications(context)
# Code linting
if self.context.pr_id and self.spec.linters:
lint = LintProcessor(self.context, self.spec, self.clone_path)
lint.process()
self.cleanup()
def clone_repository(self):
self.clone_path = os.path.join(tempfile.gettempdir(), 'badwolf',
self.task_id, self.repo_name)
source_repo = self.context.source['repository']['full_name']
# Use shallow clone to speed up
bitbucket.clone(source_repo,
self.clone_path,
depth=50,
branch=self.branch)
gitcmd = git.Git(self.clone_path)
if self.context.target:
# Pull Request
target_repo = self.context.target['repository']['full_name']
target_branch = self.context.target['branch']['name']
if source_repo == target_repo:
target_remote = 'origin'
else:
# Pull Reuqest across forks
target_remote = target_repo.split('/', 1)[0]
gitcmd.remote('add', target_remote,
bitbucket.get_git_url(target_repo))
gitcmd.fetch(target_remote, target_branch)
gitcmd.checkout('FETCH_HEAD')
gitcmd.merge('origin/{}'.format(self.branch))
else:
# Push to branch or ci retry comment on some commit
logger.info('Checkout commit %s', self.commit_hash)
gitcmd.checkout(self.commit_hash)
gitmodules = os.path.join(self.clone_path, '.gitmodules')
if os.path.exists(gitmodules):
gitcmd.submodule('update', '--init', '--recursive')
def validate_settings(self):
conf_file = os.path.join(self.clone_path,
current_app.config['BADWOLF_PROJECT_CONF'])
if not os.path.exists(conf_file):
logger.warning('No project configuration file found for repo: %s',
self.repo_full_name)
return False
self.spec = spec = Specification.parse_file(conf_file)
if self.context.type == 'commit' and spec.branch and self.branch not in spec.branch:
logger.info(
'Ignore tests since branch %s test is not enabled. Allowed branches: %s',
self.branch, spec.branch)
return False
if not spec.scripts and not spec.linters:
logger.warning('No script(s) or linter(s) to run')
return False
return True
def get_docker_image(self):
docker_image_name = self.repo_full_name.replace('/', '-')
output = []
with self.lock:
docker_image = self.docker.images(docker_image_name)
if not docker_image or self.context.rebuild:
dockerfile = os.path.join(self.clone_path,
self.spec.dockerfile)
build_options = {
'tag': docker_image_name,
'rm': True,
}
if not os.path.exists(dockerfile):
logger.warning(
'No Dockerfile: %s found for repo: %s, using simple runner image',
dockerfile, self.repo_full_name)
dockerfile_content = 'FROM messense/badwolf-test-runner\n'
fileobj = io.BytesIO(dockerfile_content.encode('utf-8'))
build_options['fileobj'] = fileobj
else:
build_options['dockerfile'] = self.spec.dockerfile
build_success = False
logger.info('Building Docker image %s', docker_image_name)
self.update_build_status('INPROGRESS', 'Building Docker image')
res = self.docker.build(self.clone_path, **build_options)
for line in res:
if b'Successfully built' in line:
build_success = True
log = to_text(json.loads(to_text(line))['stream'])
output.append(log)
logger.info('`docker build` : %s', log.strip())
if not build_success:
return None, ''.join(output)
return docker_image_name, ''.join(output)
def run_tests_in_container(self, docker_image_name):
command = '/bin/sh -c badwolf-run'
environment = {}
if self.spec.environments:
# TODO: Support run in multiple environments
environment = self.spec.environments[0]
# TODO: Add more test context related env vars
environment.update({
'DEBIAN_FRONTEND': 'noninteractive',
'CI': 'true',
'CI_NAME': 'badwolf',
'BADWOLF_BRANCH': self.branch,
'BADWOLF_COMMIT': self.commit_hash,
'BADWOLF_BUILD_DIR': '/mnt/src',
'BADWOLF_REPO_SLUG': self.repo_full_name,
})
if self.context.pr_id:
environment['BADWOLF_PULL_REQUEST'] = to_text(self.context.pr_id)
container = self.docker.create_container(
docker_image_name,
command=command,
environment=environment,
working_dir='/mnt/src',
volumes=['/mnt/src'],
host_config=self.docker.create_host_config(
privileged=self.spec.privileged,
binds={
self.clone_path: {
'bind': '/mnt/src',
'mode': 'rw',
},
}))
container_id = container['Id']
logger.info('Created container %s from image %s', container_id,
docker_image_name)
output = []
try:
self.docker.start(container_id)
self.update_build_status('INPROGRESS',
'Running tests in Docker container')
for line in self.docker.logs(container_id, stream=True):
output.append(to_text(line))
exit_code = self.docker.wait(
container_id, current_app.config['DOCKER_RUN_TIMEOUT'])
except (APIError, DockerException, ReadTimeout) as e:
exit_code = -1
output.append(to_text(e))
logger.exception('Docker error')
finally:
try:
self.docker.remove_container(container_id, force=True)
except (APIError, DockerException):
logger.exception('Error removing docker container')
return exit_code, ''.join(output)
def update_build_status(self, state, description=None):
try:
self.build_status.update(state, description=description)
except BitbucketAPIError:
logger.exception('Error calling Bitbucket API')
def send_notifications(self, context):
exit_code = context['exit_code']
template = 'test_success' if exit_code == 0 else 'test_failure'
html = render_template('mail/' + template + '.html', **context)
html = sanitize_sensitive_data(html)
# Save log html
log_dir = os.path.join(current_app.config['BADWOLF_LOG_DIR'],
self.commit_hash)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
log_file = os.path.join(log_dir, 'build.html')
with open(log_file, 'wb') as f:
f.write(to_binary(html))
if exit_code == 0:
subject = 'Test succeed for repository {}'.format(
self.repo_full_name)
else:
subject = 'Test failed for repository {}'.format(
self.repo_full_name)
notification = self.spec.notification
emails = notification['emails']
if emails:
send_mail(emails, subject, html)
slack_webhooks = notification['slack_webhooks']
if slack_webhooks:
message = render_template('slack_webhook/' + template + '.md',
**context)
trigger_slack_webhook(slack_webhooks, message)
def cleanup(self):
shutil.rmtree(os.path.dirname(self.clone_path), ignore_errors=True)
class Docker_interface:
def __init__(self,
net_name='tosker_net',
tmp_dir='/tmp',
socket='unix://var/run/docker.sock'):
self._log = Logger.get(__name__)
self._net_name = net_name
self._cli = Client(base_url=os.environ.get('DOCKER_HOST') or socket)
self._tmp_dir = tmp_dir
# TODO: aggiungere un parametro per eliminare i container se esistono gia'!
def create(self, con, cmd=None, entrypoint=None, saved_image=False):
def create_container():
tmp_dir = path.join(self._tmp_dir, con.name)
try:
os.makedirs(tmp_dir)
except:
pass
saved_img_name = '{}/{}'.format(self._net_name, con.name)
img_name = con.image
if saved_image and self.inspect(saved_img_name):
img_name = saved_img_name
self._log.debug('container: {}'.format(con.get_str_obj()))
con.id = self._cli.create_container(
name=con.name,
image=img_name,
entrypoint=entrypoint if entrypoint else con.entrypoint,
command=cmd if cmd else con.cmd,
environment=con.env,
detach=True,
# stdin_open=True,
ports=[key for key in con.ports.keys()] if con.ports else None,
volumes=['/tmp/dt'] +
([k for k, v in con.volume.items()] if con.volume else []),
networking_config=self._cli.create_networking_config({
self._net_name:
self._cli.create_endpoint_config(links=con.link
# ,aliases=['db']
)
}),
host_config=self._cli.create_host_config(
port_bindings=con.ports,
# links=con.link,
binds=[tmp_dir + ':/tmp/dt'] +
([v + ':' + k
for k, v in con.volume.items()] if con.volume else []),
)).get('Id')
assert isinstance(con, Container)
if con.to_build:
self._log.debug('start building..')
# utility.print_json(
self._cli.build(path='/'.join(con.dockerfile.split('/')[0:-1]),
dockerfile='./' + con.dockerfile.split('/')[-1],
tag=con.image,
pull=True,
quiet=True)
# )
self._log.debug('stop building..')
elif not saved_image:
# TODO: da evitare se si deve utilizzare un'immagine custom
self._log.debug('start pulling.. {}'.format(con.image))
utility.print_json(self._cli.pull(con.image, stream=True),
self._log.debug)
self._log.debug('end pulling..')
try:
create_container()
except errors.APIError as e:
self._log.debug(e)
# self.stop(con)
self.delete(con)
create_container()
# raise e
def stop(self, container):
name = self._get_name(container)
try:
return self._cli.stop(name)
except errors.NotFound as e:
self._log.error(e)
def start(self, container, wait=False):
name = self._get_name(container)
self._cli.start(name)
if wait:
self._log.debug('wait container..')
self._cli.wait(name)
utility.print_byte(self._cli.logs(name, stream=True),
self._log.debug)
def delete(self, container):
name = self._get_name(container)
try:
self._cli.remove_container(name, v=True)
except (errors.NotFound, errors.APIError) as e:
self._log.error(e)
raise e
def exec_cmd(self, container, cmd):
name = self._get_name(container)
if not self.is_running(name):
return False
try:
exec_id = self._cli.exec_create(name, cmd)
status = self._cli.exec_start(exec_id)
# TODO: verificare attendibilita' di questo check!
check = 'rpc error:' != status[:10].decode("utf-8")
self._log.debug('check: {}'.format(check))
return check
except errors.APIError as e:
self._log.error(e)
return False
except requests.exceptions.ConnectionError as e:
# TODO: questo errore arriva dopo un timeout di 10 secodi
self._log.error(e)
return False
def create_volume(self, volume):
assert isinstance(volume, Volume)
self._log.debug('volume opt: {}'.format(volume.get_all_opt()))
return self._cli.create_volume(volume.name, volume.driver,
volume.get_all_opt())
def delete_volume(self, volume):
name = self._get_name(volume)
return self._cli.remove_volume(name)
def get_containers(self, all=False):
return self._cli.containers(all=all)
def get_volumes(self):
volumes = self._cli.volumes()
return volumes['Volumes'] or []
def inspect(self, item):
name = self._get_name(item)
try:
return self._cli.inspect_container(name)
except errors.NotFound:
pass
try:
return self._cli.inspect_image(name)
except errors.NotFound:
pass
try:
return self._cli.inspect_volume(name)
except errors.NotFound:
return None
def remove_all_containers(self):
for c in self.get_containers(all=True):
self.stop(c['Id'])
self.delete(c['Id'])
def remove_all_volumes(self):
for v in self.get_volumes():
self.delete_volume(v['Name'])
def create_network(self, name, subnet='172.25.0.0/16'):
# docker network create -d bridge --subnet 172.25.0.0/16 isolated_nw
# self.delete_network(name)
try:
self._cli.create_network(name=name,
driver='bridge',
ipam={'subnet': subnet},
check_duplicate=True)
except errors.APIError:
self._log.debug('network already exists!')
def delete_network(self, name):
assert isinstance(name, str)
try:
self._cli.remove_network(name)
except errors.APIError:
self._log.debug('network not exists!')
def delete_image(self, name):
assert isinstance(name, str)
try:
self._cli.remove_image(name)
except errors.NotFound:
pass
# TODO: splittare questo metodo in due, semantica non chiara!
def update_container(self, node, cmd, saved_image=True):
assert isinstance(node, Container)
# self._log.debug('container_conf: {}'.format(node.host_container))
stat = self.inspect(node.image)
old_cmd = stat['Config']['Cmd'] or None
old_entry = stat['Config']['Entrypoint'] or None
if self.inspect(node):
self.stop(node)
self.delete(node)
self.create(node, cmd=cmd, entrypoint='', saved_image=saved_image)
self.start(node.id, wait=True)
self.stop(node.id)
name = '{}/{}'.format(self._net_name, node.name)
self._cli.commit(node.id, name)
self.stop(node)
self.delete(node)
self.create(node,
cmd=node.cmd or old_cmd,
entrypoint=node.entrypoint or old_entry,
saved_image=True)
self._cli.commit(node.id, name)
def is_running(self, container):
name = self._get_name(container)
stat = self.inspect(name)
stat = stat is not None and stat['State']['Running'] is True
self._log.debug('State: {}'.format(stat))
return stat
def _get_name(self, name):
if isinstance(name, six.string_types):
return name
else:
assert isinstance(name, (Container, Volume))
return name.name
def buildDockerfile(self):
encDockerfile = BytesIO(self.dockerfile.encode('utf-8'))
cli = Client(base_url='tcp://192.168.1.20:2375')
response = [line for line in cli.build(fileobj=encDockerfile, rm=True, tag='webpy' )]
print response
class Deploy():
def __init__(self):
self.registry_username = self.get_env("KS_DOCKER_REGISTRY_USERNAME")
self.registry_password = self.get_env("KS_DOCKER_REGISTRY_PASSWORD")
self.registry = self.get_env('KS_DOCKER_REGISTRY')
self.project_status = self.get_env("KS_PROJECT_STATUS")
self.project_name = self.get_env('KS_PROJECT_NAME')
self.project_group = self.get_env('KS_PROJECT_GROUP')
self.go_pipeline_label = self.get_env("GO_PIPELINE_LABEL")
self.project_replicas = int(self.get_env("KS_PROJECT_REPLICAS",1))
self.project_port = int(self.get_env('KS_PROJECT_PORT', 80))
self.docker_server = self.get_env("KS_DOCKER_SERVER")
self.k8s_host = self.get_env('K8S_HOST')
self.k8s_username = self.get_env('K8S_USERNAME')
self.k8s_password = self.get_env('K8S_PASSWORD')
self.container_cpu = self.get_env('KS_CONTAINER_CPU', '2')
self.container_cpu = self.get_env('KS_CONTAINER_MEMORY', '512M')
self.k8s_namespace = self.get_env('KS_PROJECT_GROUP',None)
self.go_environment_name = self.get_env('GO_ENVIRONMENT_NAME')
self.project_branch = self.get_project_branch()
self.project_commit = self.get_project_commit()
self.project_version = '{branch}-{label}.git{commit}'.format(
branch = self.project_branch,
label = self.go_pipeline_label,
commit = self.project_commit
)
self.image_name = self.project_name
self.image_tag = self.project_version
self.image_tag_name='{image}:{tag}'.format(
image = self.image_name,
tag = self.image_tag
)
self.k8s_deployment_name = self.project_name + '-' + self.project_status
self.k8s_image = self.registry + '/' + self.image_name + ':' + self.image_tag
def connect_docker(self):
self.dockerClient = Client(base_url=self.docker_server)
self.dockerClient.login(username=self.registry_username, password=self.registry_password,registry=self.registry)
def get_project_commit(self):
commit = commands.getoutput('git log --pretty=format:"%h" -n 1')
return commit
def get_project_branch(self):
branch = commands.getoutput('git branch | grep "*" | head -n 1')
branch = re.sub('\*','',branch)
branch = re.sub("/",".",branch)
branch = branch.strip()
log.info(branch)
return branch
def get_env(self,key,value=None):
value = os.getenv(key,value)
if value == None:
raise Exception('{key} not be null.'.format(key=key))
return value
def build(self, dockerfile='.'):
self.connect_docker()
image_tag = self.image_tag_name
status = True
log.info('Start To Build Image %s' % image_tag)
msgs = ''
for msg in self.dockerClient.build(path=dockerfile, tag=image_tag, forcerm=True,buildargs={'KS_PROJECT_NAME':self.project_name},nocache=True):
log.info(msg)
msgs+=msg
if re.search("error", msg, re.IGNORECASE):
status = False
if not re.search("Successfully", msgs, re.IGNORECASE):
status = False
else:
status = True
if status == True:
log.info("Build Docker Image[SUCCESS].")
else:
log.error("Build Docker Image[FAILD].")
exit(1)
return status
def push(self,image_tag_src=None,image_tag_dest=None):
self.connect_docker()
if self.registry=="" or self.registry==None:return
if image_tag_src == None:image_tag_src= self.image_tag_name
if image_tag_dest == None:image_tag_dest= self.image_tag_name
self.dockerClient.login(self.registry_username,password=self.registry_password,registry=self.registry)
self.dockerClient.tag(image_tag_src , repository=self.registry+"/"+self.image_name,tag=self.image_tag)
response = self.dockerClient.push(repository=self.registry+"/"+image_tag_dest)
if re.search("ERROR",response,re.IGNORECASE):
log.error("PUSH IMAGE[%s] TO REPOSITORY FAILD."%image_tag_dest)
log.error(response)
exit(1)
else:
log.info(response)
def deployment_action(self):
cfg_cert = K8sConfig(kubeconfig=None,auth=(self.k8s_username,self.k8s_password), api_host=self.k8s_host)
deployment = K8sDeployment(config=cfg_cert, name=self.k8s_deployment_name)
deployment.add_label(k='project',v=self.project_name)
if not deployment._exists():
container = K8sContainer(name=self.k8s_deployment_name, image=self.k8s_image)
container.add_env('KS_PROJECT_VERSION',self.project_version)
container.add_image_pull_policy(policy='Always')
container.add_volume_mount(K8sVolumeMount(name="{project}-resources-volume".format(project=self.k8s_deployment_name),
mount_path="/data/{project}/resources".format(
project=self.project_name)))
#container.add_volume_mount(K8sVolumeMount(name='localtime-volumn',mount_path='/etc/localtime'))
container.add_volume_mount(K8sVolumeMount(name='{project}-log-volume'.format(project=self.k8s_deployment_name),
mount_path='/data/logs/'))
container.add_container_resources_limits(cpu='2', memory='1024Mi')
#container.add_container_resources_requests(cpu='200m',memory='512M')
deployment.add_container(container)
deployment.add_change_cause('Image Version:{version}'.format(version=self.k8s_image))
deployment.add_image_pull_secrets([{'name': 'ksszregistrykey'}])
vol = K8sVolume(name="{project}-resources-volume".format(project=self.k8s_deployment_name), type='hostPath')
vol.path = "/data/docker/resources/{env}/{project}".format(env=self.project_status,project=self.project_name)
deployment.add_volume(vol)
#vol_localtime = K8sVolume(name='localtime-volumn', type='hostPath')
#vol_localtime.path = "/etc/localtime"
#deployment.add_volume(vol_localtime)
vol_log = K8sVolume(name="{project}-log-volume".format(project=self.k8s_deployment_name), type='hostPath')
vol_log.path = "/data/docker/logs/{env}/{project}".format(env=self.project_status,project=self.project_name)
deployment.add_volume(vol_log)
deployment.create()
else:
deployment.change_container_resources_limits(cpu='2', memory='1024Mi')
deployment.set_container_image(name=self.k8s_deployment_name, image=self.k8s_image)
deployment.add_change_cause('Image Version:{version}'.format(version=self.k8s_image))
deployment.update()
deployment.scale(self.project_replicas)
def service_action(self):
self.k8s_svc_name = self.project_name+'-'+self.project_status
cfg_cert = K8sConfig(kubeconfig=None,auth=(self.k8s_username,self.k8s_password), api_host=self.k8s_host)
service = K8sService(config=cfg_cert, name=self.k8s_svc_name)
if not service._exists():
service.add_selector(selector={"name": self.k8s_deployment_name})
service.type = 'NodePort'
service.add_port(name=self.k8s_svc_name, port=self.project_port, target_port=self.project_port, protocol='TCP')
service.create()
else:
service.add_selector(selector={"name": self.k8s_deployment_name})
service.type = 'NodePort'
if not service._exists_port(self.project_port):
service.del_port()
service.add_port(name=self.k8s_svc_name, port=self.project_port, target_port=self.project_port, protocol='TCP')
service.update()
