def build_push(self, force=False, dev=False, push=False, no_cache=False):
"""
Builds a user service, if changed, and pushes to repo if requested
"""
builder = DockerBuild(push, no_cache)
self.dev = dev
dockerfile = '.Dockerfile'
if force or not self.image_exists or self.image_stale():
log.debug("Image not found or stale - building...")
# run barrister and copy shim
rpc.generate_contract()
self.stack.copy_shim()
try:
builder.gen_dockerfile('Dockerfile-service.txt', dict(service=self), dockerfile)
builder.build_dockerfile(self.full_name, dockerfile)
finally:
self.stack.del_shim()
log.info("{} build complete".format(self.short_name))
else:
log.info("Build not necessary, run with '--force' to override")
builder.push_image(self.full_name)
def create_methods(self):
with open(CONTRACTFILE, "r") as f:
contract = json.load(f)
# remove the common.barrister element
interfaces = [x for x in contract if "barrister_version" not in x and x["type"] == "interface"]
def render_param(param):
# main render function
array_t = "[]" if param.get("is_array") else ""
name_type_t = param_t = "{}{}".format(array_t, param["type"])
if "name" in param:
return "{} {}".format(param["name"], name_type_t)
else:
return name_type_t
def render_params(params):
return [render_param(p) for p in params]
def render_signature(method):
params_t = str.join(", ", render_params(method["params"]))
return "{}({}) {}".format(method["name"], params_t, render_param(method["returns"]))
for i in interfaces:
for f in i["functions"]:
f["signature"] = render_signature(f)
log.debug('Signature for {}.{} is "{}"'.format(i["name"], f["name"], f["signature"]))
return interfaces
def __init__(self, _exit, verbose):
# setup_client
try:
# setup client depending if running on linux or using boot2docker (osx/win)
if sys.platform == 'linux':
self.client = docker_py.Client(version='auto')
else:
# get b2d ip
self.ip = str(sh.boot2docker.ip()).strip()
try:
# try secure connection first
kw = kwargs_from_env(assert_hostname=False)
self.client = docker_py.Client(version='auto', **kw)
except docker_py.errors.DockerException as e:
# shit - some weird boot2docker, python, docker-py, requests, and ssl error
# https://github.com/docker/docker-py/issues/465
if verbose:
log.debug(e)
log.warn("Cannot connect securely to Docker, trying insecurely")
kw = kwargs_from_env(assert_hostname=False)
if 'tls' in kw:
kw['tls'].verify = False
self.client = docker_py.Client(version='auto', **kw)
except Exception as e:
if verbose:
log.error("Could not connect to Docker - try running 'docker info' first")
if sys.platform != 'linux':
log.error("Please ensure you've run 'boot2docker up' and 'boot2docker shellinit' first and have added the ENV VARs it suggests")
if _exit:
raise e
def start(self, usercfg):
self.send_analytics = usercfg.send_analytics
if self.send_analytics:
log.debug("User analytics enabled")
self.analytics_ids = usercfg.analytics_ids
super().start()
else:
log.debug("User analytics disabled")
def __exit__(self, exc_type, exc_val, exc_tb):
log.debug("Shutting down Local backend")
# wait for queues to empty
self.req_q.join()
self.resp_q.join()
# change the results owner
if self.uid_gid is not None:
sh.chown('-R', self.uid_gid, self.local_store)
def stackhut_api_call(endpoint, msg, secure=True):
url = urllib.parse.urljoin(utils.SERVER_URL, endpoint)
log.debug("Calling Stackhut Server at {} with \n\t{}".format(url, json.dumps(msg)))
r = requests.post(url, data=json.dumps(msg), headers=json_header)
if r.status_code == requests.codes.ok:
return r.json()
else:
log.error("Error {} talking to Stackhut Server".format(r.status_code))
log.error(r.text)
r.raise_for_status()
def image_stale(self):
"""Runs the build only if a file has changed"""
max_mtime = self._files_mtime()
image_info = get_docker().client.inspect_image(self.full_name)
image_build_string = image_info['Created']
log.debug("Service {} last built at {}".format(self.short_name, image_build_string))
build_date = arrow.get(image_build_string).datetime.timestamp()
log.debug("Files max mtime is {}, image build date is {}".format(max_mtime, build_date))
return max_mtime >= build_date
def push_image(self, tag):
if self.push:
log.info("Uploading image {} - this may take a while...".format(tag))
with t_utils.Spinner():
r = self.docker.client.push(tag, stream=True)
r_summary = [json.loads(x.decode('utf-8')) for x in r][-1]
if 'error' in r_summary:
log.error(r_summary['error'])
log.error(r_summary['errorDetail'])
raise RuntimeError("Error pushing to Docker Hub, check your connection and auth details")
else:
log.debug(r_summary['status'])
def http_status_code(data):
if type(data) == list:
log.debug("Shit, HTTP status code incorrect")
if 'error' not in data.get('response', {}):
return 200
code = data['response']['error']['code']
if code == -32600:
return 400
elif code == -32601:
return 404
else:
return 500
def build_dockerfile(self, tag, dockerfile='Dockerfile'):
log.debug("Running docker build for {}".format(tag))
cache_flag = '--no-cache=True' if self.no_cache else '--no-cache=False'
cmds = ['-f', dockerfile, '-t', tag, '--rm', cache_flag, '.']
log.info("Starting build, this may take some time, please wait...")
try:
if utils.VERBOSE:
self.docker.run_docker_sh('build', cmds, _out=lambda x: log.debug(x.strip()))
else:
self.docker.run_docker_sh('build', cmds)
except sh.ErrorReturnCode as e:
log.error("Couldn't complete build")
log.error("Build error - {}".format(e.stderr.decode('utf-8').strip()))
if not utils.VERBOSE:
log.error("Build Traceback - \n{}".format(e.stdout.decode('utf-8').strip()))
raise RuntimeError("Docker Build failed") from None
def run(self):
while True:
(endpoint, msg) = self.queue.get()
msg.update(self.analytics_ids)
try:
log.debug("Sending analytics msg to {}".format(endpoint))
# log.debug("Analytics msg - {}".format(msg))
url = self.keen_url.format(event_collection=endpoint)
r = requests.post(url, data=json.dumps(msg), headers=json_header, timeout=2)
if not (r.status_code == requests.codes.created and r.json().get('created')):
log.debug("{} - {}".format(r.status_code, r.text()))
raise IOError()
except:
log.debug("Failed sending analytics msg to '{}'".format(endpoint))
finally:
self.queue.task_done()
def max_mtime_dir(dirname):
"""find max mtime of a single file in dir recurseively"""
for (dirpath, dirnames, fnames) in os.walk(dirname):
log.debug("Walking dir {}".format(dirname))
return max_mtime(dirpath, fnames)
def get_baseos_stack_pkgs(base_os, stack):
log.debug("OS / Stack combo for {}/{} not implemented".format(base_os.name, stack.name))
return None
def on_run_files(self, request):
log.debug("In run_files endpoint")
raise ImATeapot()
def run(self):
super().run()
# validation checks
self.usercfg.assert_logged_in()
service = Service(self.hutcfg, self.usercfg.username)
# run the contract regardless
rpc.generate_contract()
if self.local:
# call build+push first using Docker builder
if not self.no_build:
service.build_push(force=self.force, push=True, dev=self.dev)
else:
import tempfile
import requests
import os.path
from stackhut_common import utils
from stackhut_client import client
log.info("Starting Remote build, this may take a while (especially the first time), please wait...")
# compress and upload the service
with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as f:
f.close()
# get the upload url
r_file = stackhut_api_user_call("file", dict(filename=os.path.basename(f.name)), self.usercfg)
sh.tar(
"-czvf",
f.name,
"--exclude",
".git",
"--exclude",
"__pycache__",
"--exclude",
"run_result",
"--exclude",
".stackhut",
".",
)
log.debug("Uploading package {} ({:.2f} Kb)...".format(f.name, os.path.getsize(f.name) / 1024))
with open(f.name, "rb") as f1:
with Spinner():
r = requests.put(r_file["url"], data=f1)
r.raise_for_status()
# remove temp file
os.unlink(f.name)
# call the remote build service
auth = client.SHAuth(self.usercfg.username, hash=self.usercfg["hash"])
sh_client = client.SHService("stackhut", "stackhut", auth=auth, host=utils.SERVER_URL)
log.debug("Uploaded package, calling remote build...")
try:
with Spinner():
r = sh_client.Default.remoteBuild(r_file["key"], self.dev)
except client.SHRPCError as e:
log.error("Build error, remote build output below...")
log.error(e.data["output"])
return 1
else:
log.debug("Remote build output...\n" + r["cmdOutput"])
if not r["success"]:
raise RuntimeError("Build failed")
log.info("...completed Remote build")
# Inform the SH server re the new/updated service
# build up the deploy message body
test_request = json.loads(self._read_file("test_request.json"))
readme = self._read_file("README.md")
data = {
"service": service.short_name, # StackHut Service,
"github_url": self.hutcfg.github_url,
"example_request": test_request,
"description": self.hutcfg.description,
"private_service": self.hutcfg.private,
"readme": readme,
"schema": self.create_methods(),
}
log.info("Deploying image '{}' to StackHut".format(service.short_name))
r = stackhut_api_user_call("add", data, self.usercfg)
log.info("Service {} has been {} and is live".format(service.short_name, r["message"]))
log.info("You can now call this service with our client-libs or directly over JSON+HTTP")
return 0
def run(self):
super().run()
# Docker builder (if needed)
service = Service(self.hutcfg, self.usercfg.username)
service.build_push(force=self.force)
host_store_dir = os.path.abspath(LocalBackend.local_store)
os.mkdir(host_store_dir) if not os.path.exists(host_store_dir) else None
# docker setup
docker = get_docker()
log.info(
"Running service '{}' on http://{}:{}".format(
self.hutcfg.service_short_name(self.usercfg.username), docker.ip, self.port
)
)
# call docker to run the same command but in the container
# use data vols for response output files
# NOTE - SELINUX issues - can remove once Docker 1.7 becomes mainstream
import random
name = "stackhut-{}".format(random.randrange(10000))
res_flag = "z" if OS_TYPE == "SELINUX" else "rw"
verbose_mode = "-v" if self.args.verbose else None
uid_gid = "{}:{}".format(os.getuid(), os.getgid())
args = [
"-p",
"{}:4001".format(self.port),
"-v",
"{}:/workdir/{}:{}".format(host_store_dir, LocalBackend.local_store, res_flag),
"--rm=true",
"--name={}".format(name),
"--privileged" if self.args.privileged else None,
"--entrypoint=/usr/bin/env",
service.full_name,
"stackhut-runner",
verbose_mode,
"runcontainer",
"--uid",
uid_gid,
"--author",
self.usercfg.username,
]
args = [x for x in args if x is not None]
log.info("**** START SERVICE LOG ****")
try:
out = docker.run_docker_sh("run", args, _out=lambda x: print(x, end=""))
# if self.reqfile:
# host_req_file = os.path.abspath(self.reqfile)
# log.debug("Send file using reqs here")
except KeyboardInterrupt:
log.debug("Shutting down service container, press again to force-quit...")
# out.kill()
docker.run_docker_sh("stop", ["-t", "5", name])
log.info("**** END SERVICE LOG ****")
log.info("Run completed successfully")
return 0
def run_docker_sh(docker_cmd, docker_args=None, **kwargs):
_docker_args = docker_args if docker_args is not None else []
_docker_args.insert(0, docker_cmd)
log.debug("Running 'docker {}' with args {}".format(docker_cmd, _docker_args[1:]))
return sh.docker(_docker_args, **kwargs)
def __init__(self, hutcfg, author):
self.author = author
self.service_short_name = hutcfg.service_short_name(self.author)
os.mkdir(STACKHUT_DIR) if not os.path.exists(STACKHUT_DIR) else None
self.request = {}
log.debug("Starting service {}".format(self.service_short_name))
