def build_push(self, force=False, dev=False, push=False, no_cache=False): """ Builds a user service, if changed, and pushes to repo if requested """ builder = DockerBuild(push, no_cache) self.dev = dev dockerfile = '.Dockerfile' if force or not self.image_exists or self.image_stale(): log.debug("Image not found or stale - building...") # run barrister and copy shim rpc.generate_contract() self.stack.copy_shim() try: builder.gen_dockerfile('Dockerfile-service.txt', dict(service=self), dockerfile) builder.build_dockerfile(self.full_name, dockerfile) finally: self.stack.del_shim() log.info("{} build complete".format(self.short_name)) else: log.info("Build not necessary, run with '--force' to override") builder.push_image(self.full_name)
def create_methods(self): with open(CONTRACTFILE, "r") as f: contract = json.load(f) # remove the common.barrister element interfaces = [x for x in contract if "barrister_version" not in x and x["type"] == "interface"] def render_param(param): # main render function array_t = "[]" if param.get("is_array") else "" name_type_t = param_t = "{}{}".format(array_t, param["type"]) if "name" in param: return "{} {}".format(param["name"], name_type_t) else: return name_type_t def render_params(params): return [render_param(p) for p in params] def render_signature(method): params_t = str.join(", ", render_params(method["params"])) return "{}({}) {}".format(method["name"], params_t, render_param(method["returns"])) for i in interfaces: for f in i["functions"]: f["signature"] = render_signature(f) log.debug('Signature for {}.{} is "{}"'.format(i["name"], f["name"], f["signature"])) return interfaces
def __init__(self, _exit, verbose): # setup_client try: # setup client depending if running on linux or using boot2docker (osx/win) if sys.platform == 'linux': self.client = docker_py.Client(version='auto') else: # get b2d ip self.ip = str(sh.boot2docker.ip()).strip() try: # try secure connection first kw = kwargs_from_env(assert_hostname=False) self.client = docker_py.Client(version='auto', **kw) except docker_py.errors.DockerException as e: # shit - some weird boot2docker, python, docker-py, requests, and ssl error # https://github.com/docker/docker-py/issues/465 if verbose: log.debug(e) log.warn("Cannot connect securely to Docker, trying insecurely") kw = kwargs_from_env(assert_hostname=False) if 'tls' in kw: kw['tls'].verify = False self.client = docker_py.Client(version='auto', **kw) except Exception as e: if verbose: log.error("Could not connect to Docker - try running 'docker info' first") if sys.platform != 'linux': log.error("Please ensure you've run 'boot2docker up' and 'boot2docker shellinit' first and have added the ENV VARs it suggests") if _exit: raise e
def start(self, usercfg): self.send_analytics = usercfg.send_analytics if self.send_analytics: log.debug("User analytics enabled") self.analytics_ids = usercfg.analytics_ids super().start() else: log.debug("User analytics disabled")
def __exit__(self, exc_type, exc_val, exc_tb): log.debug("Shutting down Local backend") # wait for queues to empty self.req_q.join() self.resp_q.join() # change the results owner if self.uid_gid is not None: sh.chown('-R', self.uid_gid, self.local_store)
def stackhut_api_call(endpoint, msg, secure=True): url = urllib.parse.urljoin(utils.SERVER_URL, endpoint) log.debug("Calling Stackhut Server at {} with \n\t{}".format(url, json.dumps(msg))) r = requests.post(url, data=json.dumps(msg), headers=json_header) if r.status_code == requests.codes.ok: return r.json() else: log.error("Error {} talking to Stackhut Server".format(r.status_code)) log.error(r.text) r.raise_for_status()
def image_stale(self): """Runs the build only if a file has changed""" max_mtime = self._files_mtime() image_info = get_docker().client.inspect_image(self.full_name) image_build_string = image_info['Created'] log.debug("Service {} last built at {}".format(self.short_name, image_build_string)) build_date = arrow.get(image_build_string).datetime.timestamp() log.debug("Files max mtime is {}, image build date is {}".format(max_mtime, build_date)) return max_mtime >= build_date
def push_image(self, tag): if self.push: log.info("Uploading image {} - this may take a while...".format(tag)) with t_utils.Spinner(): r = self.docker.client.push(tag, stream=True) r_summary = [json.loads(x.decode('utf-8')) for x in r][-1] if 'error' in r_summary: log.error(r_summary['error']) log.error(r_summary['errorDetail']) raise RuntimeError("Error pushing to Docker Hub, check your connection and auth details") else: log.debug(r_summary['status'])
def http_status_code(data): if type(data) == list: log.debug("Shit, HTTP status code incorrect") if 'error' not in data.get('response', {}): return 200 code = data['response']['error']['code'] if code == -32600: return 400 elif code == -32601: return 404 else: return 500
def build_dockerfile(self, tag, dockerfile='Dockerfile'): log.debug("Running docker build for {}".format(tag)) cache_flag = '--no-cache=True' if self.no_cache else '--no-cache=False' cmds = ['-f', dockerfile, '-t', tag, '--rm', cache_flag, '.'] log.info("Starting build, this may take some time, please wait...") try: if utils.VERBOSE: self.docker.run_docker_sh('build', cmds, _out=lambda x: log.debug(x.strip())) else: self.docker.run_docker_sh('build', cmds) except sh.ErrorReturnCode as e: log.error("Couldn't complete build") log.error("Build error - {}".format(e.stderr.decode('utf-8').strip())) if not utils.VERBOSE: log.error("Build Traceback - \n{}".format(e.stdout.decode('utf-8').strip())) raise RuntimeError("Docker Build failed") from None
def run(self): while True: (endpoint, msg) = self.queue.get() msg.update(self.analytics_ids) try: log.debug("Sending analytics msg to {}".format(endpoint)) # log.debug("Analytics msg - {}".format(msg)) url = self.keen_url.format(event_collection=endpoint) r = requests.post(url, data=json.dumps(msg), headers=json_header, timeout=2) if not (r.status_code == requests.codes.created and r.json().get('created')): log.debug("{} - {}".format(r.status_code, r.text())) raise IOError() except: log.debug("Failed sending analytics msg to '{}'".format(endpoint)) finally: self.queue.task_done()
def max_mtime_dir(dirname): """find max mtime of a single file in dir recurseively""" for (dirpath, dirnames, fnames) in os.walk(dirname): log.debug("Walking dir {}".format(dirname)) return max_mtime(dirpath, fnames)
def get_baseos_stack_pkgs(base_os, stack): log.debug("OS / Stack combo for {}/{} not implemented".format(base_os.name, stack.name)) return None
def on_run_files(self, request): log.debug("In run_files endpoint") raise ImATeapot()
def run(self): super().run() # validation checks self.usercfg.assert_logged_in() service = Service(self.hutcfg, self.usercfg.username) # run the contract regardless rpc.generate_contract() if self.local: # call build+push first using Docker builder if not self.no_build: service.build_push(force=self.force, push=True, dev=self.dev) else: import tempfile import requests import os.path from stackhut_common import utils from stackhut_client import client log.info("Starting Remote build, this may take a while (especially the first time), please wait...") # compress and upload the service with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as f: f.close() # get the upload url r_file = stackhut_api_user_call("file", dict(filename=os.path.basename(f.name)), self.usercfg) sh.tar( "-czvf", f.name, "--exclude", ".git", "--exclude", "__pycache__", "--exclude", "run_result", "--exclude", ".stackhut", ".", ) log.debug("Uploading package {} ({:.2f} Kb)...".format(f.name, os.path.getsize(f.name) / 1024)) with open(f.name, "rb") as f1: with Spinner(): r = requests.put(r_file["url"], data=f1) r.raise_for_status() # remove temp file os.unlink(f.name) # call the remote build service auth = client.SHAuth(self.usercfg.username, hash=self.usercfg["hash"]) sh_client = client.SHService("stackhut", "stackhut", auth=auth, host=utils.SERVER_URL) log.debug("Uploaded package, calling remote build...") try: with Spinner(): r = sh_client.Default.remoteBuild(r_file["key"], self.dev) except client.SHRPCError as e: log.error("Build error, remote build output below...") log.error(e.data["output"]) return 1 else: log.debug("Remote build output...\n" + r["cmdOutput"]) if not r["success"]: raise RuntimeError("Build failed") log.info("...completed Remote build") # Inform the SH server re the new/updated service # build up the deploy message body test_request = json.loads(self._read_file("test_request.json")) readme = self._read_file("README.md") data = { "service": service.short_name, # StackHut Service, "github_url": self.hutcfg.github_url, "example_request": test_request, "description": self.hutcfg.description, "private_service": self.hutcfg.private, "readme": readme, "schema": self.create_methods(), } log.info("Deploying image '{}' to StackHut".format(service.short_name)) r = stackhut_api_user_call("add", data, self.usercfg) log.info("Service {} has been {} and is live".format(service.short_name, r["message"])) log.info("You can now call this service with our client-libs or directly over JSON+HTTP") return 0
def run(self): super().run() # Docker builder (if needed) service = Service(self.hutcfg, self.usercfg.username) service.build_push(force=self.force) host_store_dir = os.path.abspath(LocalBackend.local_store) os.mkdir(host_store_dir) if not os.path.exists(host_store_dir) else None # docker setup docker = get_docker() log.info( "Running service '{}' on http://{}:{}".format( self.hutcfg.service_short_name(self.usercfg.username), docker.ip, self.port ) ) # call docker to run the same command but in the container # use data vols for response output files # NOTE - SELINUX issues - can remove once Docker 1.7 becomes mainstream import random name = "stackhut-{}".format(random.randrange(10000)) res_flag = "z" if OS_TYPE == "SELINUX" else "rw" verbose_mode = "-v" if self.args.verbose else None uid_gid = "{}:{}".format(os.getuid(), os.getgid()) args = [ "-p", "{}:4001".format(self.port), "-v", "{}:/workdir/{}:{}".format(host_store_dir, LocalBackend.local_store, res_flag), "--rm=true", "--name={}".format(name), "--privileged" if self.args.privileged else None, "--entrypoint=/usr/bin/env", service.full_name, "stackhut-runner", verbose_mode, "runcontainer", "--uid", uid_gid, "--author", self.usercfg.username, ] args = [x for x in args if x is not None] log.info("**** START SERVICE LOG ****") try: out = docker.run_docker_sh("run", args, _out=lambda x: print(x, end="")) # if self.reqfile: # host_req_file = os.path.abspath(self.reqfile) # log.debug("Send file using reqs here") except KeyboardInterrupt: log.debug("Shutting down service container, press again to force-quit...") # out.kill() docker.run_docker_sh("stop", ["-t", "5", name]) log.info("**** END SERVICE LOG ****") log.info("Run completed successfully") return 0
def run_docker_sh(docker_cmd, docker_args=None, **kwargs): _docker_args = docker_args if docker_args is not None else [] _docker_args.insert(0, docker_cmd) log.debug("Running 'docker {}' with args {}".format(docker_cmd, _docker_args[1:])) return sh.docker(_docker_args, **kwargs)
def __init__(self, hutcfg, author): self.author = author self.service_short_name = hutcfg.service_short_name(self.author) os.mkdir(STACKHUT_DIR) if not os.path.exists(STACKHUT_DIR) else None self.request = {} log.debug("Starting service {}".format(self.service_short_name))