def build_push(self, force=False, dev=False, push=False, no_cache=False):
"""
Builds a user service, if changed, and pushes to repo if requested
"""
builder = DockerBuild(push, no_cache)
self.dev = dev
dockerfile = '.Dockerfile'
if force or not self.image_exists or self.image_stale():
log.debug("Image not found or stale - building...")
# run barrister and copy shim
rpc.generate_contract()
self.stack.copy_shim()
try:
builder.gen_dockerfile('Dockerfile-service.txt', dict(service=self), dockerfile)
builder.build_dockerfile(self.full_name, dockerfile)
finally:
self.stack.del_shim()
log.info("{} build complete".format(self.short_name))
else:
log.info("Build not necessary, run with '--force' to override")
builder.push_image(self.full_name)
def _process_request(self, data):
try:
self.request = json.loads(data.decode('utf-8'))
rpc.add_get_id(self.request)
log.info("Request - {}".format(self.request))
if ((self.request['service'] != self.service_short_name) and ((self.request['service']+':latest') != self.service_short_name)):
log.warn("Service request ({}) sent to wrong service ({})".format(self.request['service'], self.service_short_name))
except Exception as e:
_e = rpc.exc_to_json_error(rpc.ParseError(dict(exception=repr(e))))
return True, _e
else:
return False, self.request
def push_image(self, tag):
if self.push:
log.info("Uploading image {} - this may take a while...".format(tag))
with t_utils.Spinner():
r = self.docker.client.push(tag, stream=True)
r_summary = [json.loads(x.decode('utf-8')) for x in r][-1]
if 'error' in r_summary:
log.error(r_summary['error'])
log.error(r_summary['errorDetail'])
raise RuntimeError("Error pushing to Docker Hub, check your connection and auth details")
else:
log.debug(r_summary['status'])
def build_push(self, baseos, outdir, *args):
log.info("Building image for base {} with stack {}".format(baseos.name, self.name))
image_name = "{}-{}".format(baseos.name, self.name)
baseos_stack_cmds_pkgs = get_baseos_stack_pkgs(baseos, self)
if baseos_stack_cmds_pkgs is not None:
os_cmds, pkgs = baseos_stack_cmds_pkgs
pkg_cmds = baseos.install_os_pkg(pkgs) if pkgs else []
stack_cmds = os_cmds + pkg_cmds
# only render the template if apy supported config
builder = DockerBuild(*args)
builder.stack_build_push('Dockerfile-stack.txt',
dict(baseos=baseos, stack=self, stack_cmds=stack_cmds),
outdir, image_name)
def run(self):
toolkit_stack = stacks[self.hutcfg.stack]
toolkit_stack.copy_shim()
rpc.generate_contract()
try:
backend = LocalBackend(self.hutcfg, self.usercfg.username, port=self.port)
with ServiceRunner(backend, self.hutcfg) as runner:
log.info("Running service '{}' on http://127.0.0.1:{}".format(backend.service_short_name, self.port))
runner.run()
finally:
toolkit_stack.del_shim()
os.remove(rpc.REQ_FIFO)
os.remove(rpc.RESP_FIFO)
def run(self):
super().run()
# Python, y u no have assert function??
assert self.usercfg.username == "stackhut", "Must be logged in as StackHut user to build & deploy these"
# build bases and stacks
[b.build_push(self.outdir, self.args.push, self.args.no_cache) for b in bases.values()]
[
s.build_push(b, self.outdir, self.args.push, self.args.no_cache)
for b in bases.values()
for s in stacks.values()
]
log.info("All base OS and Stack images built and deployed")
return 0
def build_dockerfile(self, tag, dockerfile='Dockerfile'):
log.debug("Running docker build for {}".format(tag))
cache_flag = '--no-cache=True' if self.no_cache else '--no-cache=False'
cmds = ['-f', dockerfile, '-t', tag, '--rm', cache_flag, '.']
log.info("Starting build, this may take some time, please wait...")
try:
if utils.VERBOSE:
self.docker.run_docker_sh('build', cmds, _out=lambda x: log.debug(x.strip()))
else:
self.docker.run_docker_sh('build', cmds)
except sh.ErrorReturnCode as e:
log.error("Couldn't complete build")
log.error("Build error - {}".format(e.stderr.decode('utf-8').strip()))
if not utils.VERBOSE:
log.error("Build Traceback - \n{}".format(e.stdout.decode('utf-8').strip()))
raise RuntimeError("Docker Build failed") from None
def run(self):
super().run()
# validation checks
self.usercfg.assert_logged_in()
if os.path.exists(".git") or os.path.exists("Hutfile.yaml"):
raise RuntimeError("Found existing project, cancelling")
if not is_stack_supported(self.baseos, self.stack):
raise ValueError(
"Sorry, the combination of {} and {} is currently unsupported".format(self.baseos, self.stack)
)
# set and check service name
self.name = os.path.basename(os.getcwd())
HutfileCfg.assert_valid_name(self.name)
self.author = self.usercfg.username
log.info("Creating service {}/{}".format(self.author, self.name))
# copy the scaffolds into the service
def copy_scaffold(name):
dir_path = get_res_path(os.path.join("scaffold", name))
copy_tree(dir_path, ".")
return os.listdir(dir_path)
common_files = copy_scaffold("common")
stack_files = copy_scaffold(self.stack.name)
# run the templates
template_env = Environment(loader=FileSystemLoader("."))
[
self.render_file(template_env, f, dict(scaffold=self))
for f in (common_files + stack_files)
if os.path.exists(f) and os.path.isfile(f)
]
# git commit
if not self.args.no_git:
sh.git.init()
sh.git.add(".")
sh.git.commit(m="Initial commit")
sh.git.branch("stackhut")
return 0
def run(self):
import hashlib
super().run()
username = input("Username: "******"Email: ")
password = getpass.getpass("Password: "******"login", dict(username=username, password=password))
if r["success"]:
# self.usercfg['docker_username'] = docker_username
self.usercfg["username"] = username
self.usercfg["hash"] = r["hash"]
self.usercfg["u_id"] = hashlib.sha256(("stackhut_is_da_bomb" + username).encode("utf-8")).hexdigest()
# self.usercfg['email'] = r['email']
self.usercfg.save()
log.info("User {} logged in successfully".format(username))
else:
raise RuntimeError("Incorrect username or password, please try again")
return 0
def build_push(self, outdir, *args):
log.info("Building image for base {}".format(self.name))
image_name = self.name
builder = DockerBuild(*args)
builder.stack_build_push('Dockerfile-baseos.txt', dict(baseos=self), outdir, image_name)
def _process_response(self, data):
log.info("Response - {}".format(data))
self.request = {}
return json.dumps(data).encode('utf-8')
def run(self):
# start in a new thread
log.info("Started StackHut Request Server - press Ctrl-C to quit")
run_simple('0.0.0.0', self.port, self.local_server)
def run(self):
super().run()
# connect to Stackhut service to get hash?
log.info("Logged out {}".format(self.usercfg.get("email", "")))
self.usercfg.wipe()
return 0
def run(self):
super().run()
# validation checks
self.usercfg.assert_logged_in()
service = Service(self.hutcfg, self.usercfg.username)
# run the contract regardless
rpc.generate_contract()
if self.local:
# call build+push first using Docker builder
if not self.no_build:
service.build_push(force=self.force, push=True, dev=self.dev)
else:
import tempfile
import requests
import os.path
from stackhut_common import utils
from stackhut_client import client
log.info("Starting Remote build, this may take a while (especially the first time), please wait...")
# compress and upload the service
with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as f:
f.close()
# get the upload url
r_file = stackhut_api_user_call("file", dict(filename=os.path.basename(f.name)), self.usercfg)
sh.tar(
"-czvf",
f.name,
"--exclude",
".git",
"--exclude",
"__pycache__",
"--exclude",
"run_result",
"--exclude",
".stackhut",
".",
)
log.debug("Uploading package {} ({:.2f} Kb)...".format(f.name, os.path.getsize(f.name) / 1024))
with open(f.name, "rb") as f1:
with Spinner():
r = requests.put(r_file["url"], data=f1)
r.raise_for_status()
# remove temp file
os.unlink(f.name)
# call the remote build service
auth = client.SHAuth(self.usercfg.username, hash=self.usercfg["hash"])
sh_client = client.SHService("stackhut", "stackhut", auth=auth, host=utils.SERVER_URL)
log.debug("Uploaded package, calling remote build...")
try:
with Spinner():
r = sh_client.Default.remoteBuild(r_file["key"], self.dev)
except client.SHRPCError as e:
log.error("Build error, remote build output below...")
log.error(e.data["output"])
return 1
else:
log.debug("Remote build output...\n" + r["cmdOutput"])
if not r["success"]:
raise RuntimeError("Build failed")
log.info("...completed Remote build")
# Inform the SH server re the new/updated service
# build up the deploy message body
test_request = json.loads(self._read_file("test_request.json"))
readme = self._read_file("README.md")
data = {
"service": service.short_name, # StackHut Service,
"github_url": self.hutcfg.github_url,
"example_request": test_request,
"description": self.hutcfg.description,
"private_service": self.hutcfg.private,
"readme": readme,
"schema": self.create_methods(),
}
log.info("Deploying image '{}' to StackHut".format(service.short_name))
r = stackhut_api_user_call("add", data, self.usercfg)
log.info("Service {} has been {} and is live".format(service.short_name, r["message"]))
log.info("You can now call this service with our client-libs or directly over JSON+HTTP")
return 0
def run(self):
super().run()
# Docker builder (if needed)
service = Service(self.hutcfg, self.usercfg.username)
service.build_push(force=self.force)
host_store_dir = os.path.abspath(LocalBackend.local_store)
os.mkdir(host_store_dir) if not os.path.exists(host_store_dir) else None
# docker setup
docker = get_docker()
log.info(
"Running service '{}' on http://{}:{}".format(
self.hutcfg.service_short_name(self.usercfg.username), docker.ip, self.port
)
)
# call docker to run the same command but in the container
# use data vols for response output files
# NOTE - SELINUX issues - can remove once Docker 1.7 becomes mainstream
import random
name = "stackhut-{}".format(random.randrange(10000))
res_flag = "z" if OS_TYPE == "SELINUX" else "rw"
verbose_mode = "-v" if self.args.verbose else None
uid_gid = "{}:{}".format(os.getuid(), os.getgid())
args = [
"-p",
"{}:4001".format(self.port),
"-v",
"{}:/workdir/{}:{}".format(host_store_dir, LocalBackend.local_store, res_flag),
"--rm=true",
"--name={}".format(name),
"--privileged" if self.args.privileged else None,
"--entrypoint=/usr/bin/env",
service.full_name,
"stackhut-runner",
verbose_mode,
"runcontainer",
"--uid",
uid_gid,
"--author",
self.usercfg.username,
]
args = [x for x in args if x is not None]
log.info("**** START SERVICE LOG ****")
try:
out = docker.run_docker_sh("run", args, _out=lambda x: print(x, end=""))
# if self.reqfile:
# host_req_file = os.path.abspath(self.reqfile)
# log.debug("Send file using reqs here")
except KeyboardInterrupt:
log.debug("Shutting down service container, press again to force-quit...")
# out.kill()
docker.run_docker_sh("stop", ["-t", "5", name])
log.info("**** END SERVICE LOG ****")
log.info("Run completed successfully")
return 0
def run(self):
super().run()
# log sys info
log.info("StackHut version {}".format(__version__))
# docker info
docker = get_docker(_exit=False, verbose=False)
if docker is not None:
log.info("Docker version {}".format(docker.client.version().get("Version")))
else:
log.info("Docker not installed or connection error")
# usercfg info
if self.usercfg.logged_in:
log.info("User logged in")
for x in self.usercfg.show_keys:
log.info("{}: {}".format(x, self.usercfg.get(x)))
else:
log.info("User not logged in")
return 0