def configure_supervisor(application=None, uwsgi_args={}):
"""
Configures supervisord
:keyword application: Application to configure
"""
log = config.get_logger("configure_supervisor")
log.info("{0}: configuring supervisor".format(application))
errors = {}
output = {}
app_config = utils.get_application_config(application)
try:
app_config = json.loads(app_config)
except:
raise RuntimeError("Invalid or missing application config")
app_dir = os.path.join(settings.APPLICATION_BASE_DIR, application)
app_state_dir = os.path.join(settings.APPLICATION_STATE_DIR, application)
app_local_dir = os.listdir(app_dir)[0]
if not os.path.exists(app_state_dir):
os.makedirs(app_state_dir)
for instance in app_config["instances"][settings.NODE_NAME]:
# generate uwsgi config
supervisor_conf = os.path.join(settings.SUPERVISOR_CONF_DIR, "uwsgi_{0}_{1}.conf".format(application, instance))
uwsgi_config = "[program:uwsgi_{0}_{1}]\n".format(application, instance)
uwsgi_config += "command=uwsgi\n"
# defaults
uwsgi_config += " --uid {0}\n".format(settings.APPLICATION_USER)
uwsgi_config += " --gid {0}\n".format(settings.APPLICATION_GROUP)
uwsgi_config += " -s {0}\n".format(os.path.join(app_state_dir, "{0}_{1}.sock".format(application, instance)))
uwsgi_config += " -H {0}\n".format(os.path.join(settings.VIRTUALENV_BASE_DIR, application))
uwsgi_config += " -M\n"
uwsgi_config += " -C\n"
uwsgi_config += " -p 2\n"
uwsgi_config += " --no-orphans\n"
uwsgi_config += " --vacuum\n"
uwsgi_config += " --harakiri 300\n"
uwsgi_config += " --max-requests 5000\n"
uwsgi_config += " --python-path {0}\n".format(app_local_dir)
uwsgi_config += " -w wsgi\n"
# uwsgi args
for k, v in uwsgi_args.iteritems():
uwsgi_config += " --{0} {1}\n".format(k, v)
uwsgi_config += "directory={0}\n".format(os.path.join(settings.APPLICATION_BASE_DIR, application))
uwsgi_config += "user={0}\n".format(settings.APPLICATION_USER)
uwsgi_config += "redirect_stderr=true\n"
uwsgi_config += "stdout_logfile={0}/{1}.log\n".format(settings.SUPERVISOR_CONF_DIR, application)
uwsgi_config += "stopsignal=QUIT\n"
output["uwsgi_config_{0}".format(instance)] = uwsgi_config
# create config
with open(supervisor_conf, "w") as f:
f.write(uwsgi_config)
# signal supervisor to update
p = Popen(["supervisorctl", "update"], stdout=PIPE, stderr=PIPE)
p_out, p_err = p.stdout.read().strip(), p.stderr.read().strip()
output["supervisorctl_out"] = p_out
output["supervisorctl_err"] = p_err
data = {"status": "complete", "output": output, "errors": errors, "operation": "configure_supervisor"}
return data
def decorated(*args, **kwargs):
if 'app_uuid' in kwargs:
user = utils.get_user(session['user'])
app = utils.get_application_config(app_uuid=kwargs['app_uuid'])
if app['owner'] != user['username']:
flash(messages.ACCESS_DENIED, 'error')
return redirect(url_for('applications'))
return f(*args, **kwargs)
def configure_webserver(application=None):
"""
Configures webserver (current Nginx)
:keyword application: Application to configure
"""
log = config.get_logger("configure_webserver")
log.info("{0}: configuring webserver".format(application))
errors = {}
output = {}
app_config = utils.get_application_config(application)
try:
app_config = json.loads(app_config)
except:
raise RuntimeError("Invalid or missing application config")
app_state_dir = os.path.join(settings.APPLICATION_STATE_DIR, application)
for instance in app_config["instances"][settings.NODE_NAME]:
# generate nginx config
nginx_conf = os.path.join(settings.WEBSERVER_CONF_DIR, "nginx_{0}_{1}.conf".format(application, instance))
conf = "# {0} config for application: {1} instance {2}\n".format(settings.APP_NAME, application, instance)
conf += "user {0};\n".format(settings.APPLICATION_USER)
conf += "worker_processes 1;\n"
conf += "error_log {0};\n".format(
os.path.join(settings.APPLICATION_LOG_DIR, "{0}_{1}-error.log".format(application, "nginx"))
)
conf += "pid {0};\n".format(os.path.join(app_state_dir, "nginx_{0}_{1}.pid".format(application, instance)))
conf += "events {\n"
conf += " worker_connections 1024;\n"
conf += "}\n"
conf += "http {\n"
conf += " include mime.types;\n"
conf += " default_type application/octet-stream;\n"
conf += " sendfile on;\n"
conf += " keepalive_timeout 65;\n"
conf += " server {\n"
conf += " listen {0};\n".format(instance)
conf += " access_log {0};\n".format(
os.path.join(settings.APPLICATION_LOG_DIR, "{0}_{1}-access.log".format(application, "nginx"))
)
conf += " location /nginx-status {\n"
conf += " stub_status on;\n"
conf += " }\n"
conf += " location / {\n"
conf += " uwsgi_pass unix://{0};\n".format(
os.path.join(app_state_dir, "{0}_{1}.sock".format(application, instance))
)
conf += " include uwsgi_params;\n"
conf += " }\n"
conf += " }\n"
conf += "}\n"
# write config
with open(nginx_conf, "w") as f:
f.write(conf)
output["nginx_conf_{0}".format(instance)] = conf
data = {"status": "complete", "output": output, "errors": errors, "operation": "configure_webserver"}
return data
def application(app_uuid=None):
try:
app = utils.get_application_config(app_uuid=app_uuid)
ctx = {
'application': app,
}
return render_template('application.html', **ctx)
except Exception, e:
flash(e, 'error')
return redirect(url_for('applications'))
def restart_application(app_name=None):
"""
Restarts an application
:keyword app_name: Name of application to restart
"""
if not app_name:
raise NameError("You must specify an application name")
log = config.get_logger("restart_application")
errors = {}
output = {}
log.info("{0}: restarting application".format(app_name))
app_config = utils.get_application_config(app_name)
try:
app_config = json.loads(app_config)
except:
raise RuntimeError("Invalid or missing application config")
# HACK: chown the state dir (and existing files)
for f in glob.iglob("{0}".format(settings.APPLICATION_STATE_DIR)):
os.chown(f, pwd.getpwnam(settings.APPLICATION_USER).pw_uid, grp.getgrnam(settings.APPLICATION_GROUP).gr_gid)
for f in glob.iglob("{0}/*".format(settings.APPLICATION_STATE_DIR)):
os.chown(f, pwd.getpwnam(settings.APPLICATION_USER).pw_uid, grp.getgrnam(settings.APPLICATION_GROUP).gr_gid)
for f in glob.iglob("{0}/*/*".format(settings.APPLICATION_STATE_DIR)):
os.chown(f, pwd.getpwnam(settings.APPLICATION_USER).pw_uid, grp.getgrnam(settings.APPLICATION_GROUP).gr_gid)
# attempt to stop application first
stop_application(app_name)
# signal nginx to start
for instance in app_config["instances"][settings.NODE_NAME]:
app_nginx_conf = os.path.join(settings.WEBSERVER_CONF_DIR, "nginx_{0}_{1}.conf".format(app_name, instance))
if os.path.exists(app_nginx_conf):
p = Popen(["nginx", "-c", app_nginx_conf], stdout=PIPE, stderr=PIPE)
out, err = p.stdout.read().strip(), p.stderr.read().strip()
if out != "":
output["nginx_out_{0}".format(instance)] = out
if err != "":
output["nginx_err_{0}".format(instance)] = err
# signal uwsgi to stop
p = Popen(["supervisorctl", "start", "uwsgi_{0}_{1}".format(app_name, instance)], stdout=PIPE, stderr=PIPE)
out, err = p.stdout.read().strip(), p.stderr.read().strip()
if out != "":
output["uwsgi_out_{0}".format(instance)] = out
if err != "":
output["uwsgi_err_{0}".format(instance)] = err
data = {"status": "complete", "output": output, "errors": errors, "operation": "restart_application"}
return data
def stop_application(app_name=None):
"""
Stops an application
:keyword app_name: Application to stop
"""
if not app_name:
raise NameError("You must specify an application name")
log = config.get_logger("stop_application")
log.info("{0}: stopping application".format(app_name))
errors = {}
output = {}
app_config = utils.get_application_config(app_name)
try:
app_config = json.loads(app_config)
except:
raise RuntimeError("Invalid or missing application config")
for instance in app_config["instances"][settings.NODE_NAME]:
# signal nginx to stop
app_nginx_conf = os.path.join(settings.WEBSERVER_CONF_DIR, "nginx_{0}_{1}.conf".format(app_name, instance))
if os.path.exists(app_nginx_conf):
p = Popen(["nginx", "-c", app_nginx_conf, "-s", "quit"], stdout=PIPE, stderr=PIPE)
out, err = p.stdout.read().strip(), p.stderr.read().strip()
if out != "":
output["nginx_out_{0}".format(instance)] = out
if err != "":
output["nginx_err_{0}".format(instance)] = err
# signal uwsgi to stop
p = Popen(["supervisorctl", "stop", "uwsgi_{0}_{1}".format(app_name, instance)], stdout=PIPE, stderr=PIPE)
out, err = p.stdout.read().strip(), p.stderr.read().strip()
if out != "":
output["uwsgi_out_{0}".format(instance)] = out
if err != "":
output["uwsgi_err_{0}".format(instance)] = err
# run fuser to kill anything else
p = Popen(["fuser", "-k", os.path.join(settings.APPLICATION_BASE_DIR, app_name)], stdout=PIPE, stderr=PIPE)
out, err = p.stdout.read().strip(), p.stderr.read().strip()
if out != "":
output["fuser_out"] = out
if err != "":
output["fuser_err"] = err
data = {"status": "complete", "output": output, "errors": errors, "operation": "stop_application"}
return data
def api_app_deploy(app_uuid=None):
try:
data = {}
pkg_id = app_uuid
# check if owner
app_config = utils.get_application_config(app_uuid=app_uuid)
user = utils.get_user(apikey=session['apikey'])
if app_config['owner'] != user['username']:
data['error'] = messages.ACCESS_DENIED
status = 400
else:
f = request.files['package']
pkg_name = os.path.join(app.config['DEPLOY_WORK_DIR'], pkg_id)
f.save(pkg_name)
api_log.info('Deploy for {0} requested from {1} ({2})'.format(\
app_config['name'], user['username'], request.remote_addr))
data['task_id'] = api.deploy_package.delay(app_config['name'], pkg_name).key
status = 200
except Exception, e:
data = {'error': str(e)}
status = 400
def scale_application(app_name=None, instances=None):
"""
Scales application to number of instances
:keyword app_name: Name of application to scale
:keyword instances: Number of instances
"""
if not app_name:
raise NameError("You must specify an application name")
log = config.get_logger("scale_application")
errors = {}
output = {}
# attempt to stop application first
stop_application(app_name)
log.info("{0}: scaling application".format(app_name))
app_config = utils.get_application_config(app_name)
try:
app_config = json.loads(app_config)
except:
raise RuntimeError("Invalid or missing application config")
def index():
if 'auth_token' in session:
ctx = {}
try:
applications = utils.get_node_applications()
apps = []
for app in applications:
json_data = json.loads(utils.get_application_config(app))
app_data = {
'name': app,
'version': json_data['version'],
'instances': json_data['instances'][settings.NODE_NAME],
'runtime': json_data['runtime'],
}
apps.append(app_data)
ctx = {
'applications': apps,
}
except Exception, e:
import traceback
traceback.print_exc()
flash(e, 'error')
return render_template("index.html", **ctx)
def remove_application(app_name=None):
"""
Removes and application
:keyword app_name: Name of application to remove
"""
if not app_name:
raise NameError("You must specify an application name")
log = config.get_logger("remove_application")
errors = {}
output = {}
if app_name not in utils.get_node_applications():
raise NameError("Application not deployed to this node")
app_config = utils.get_application_config(app_name)
try:
app_config = json.loads(app_config)
# attempt to stop application first
stop_application(app_name)
except:
pass # ignore errors on removal
# remove configs
app_state_dir = os.path.join(settings.APPLICATION_STATE_DIR, app_name)
app_dir = os.path.join(settings.APPLICATION_BASE_DIR, app_name)
ve_dir = os.path.join(settings.VIRTUALENV_BASE_DIR, app_name)
if os.path.exists(app_state_dir):
shutil.rmtree(app_state_dir)
output["app_state_dir"] = "removed"
if os.path.exists(app_dir):
shutil.rmtree(app_dir)
output["app_dir"] = "removed"
if os.path.exists(ve_dir):
shutil.rmtree(ve_dir)
output["ve_dir"] = "removed"
# clear nginx configs
for conf in os.listdir(settings.WEBSERVER_CONF_DIR):
if re.search("{0}*".format(app_name), conf):
os.remove(os.path.join(settings.WEBSERVER_CONF_DIR, conf))
output["nginx_configs"] = "removed"
# clear uwsgi configs
for conf in os.listdir(settings.SUPERVISOR_CONF_DIR):
if re.search("uwsgi_{0}*".format(app_name), conf):
os.remove(os.path.join(settings.SUPERVISOR_CONF_DIR, conf))
output["uwsgi_configs"] = "removed"
# clear logs
for log_file in os.listdir(settings.APPLICATION_LOG_DIR):
if re.search("{0}*".format(app_name), log_file):
os.remove(os.path.join(settings.APPLICATION_LOG_DIR, log_file))
# get app config to remove reserved instances
if app_config:
for instance in app_config["instances"][settings.NODE_NAME]:
log.debug("{0}: releasing port: {1}".format(app_name, instance))
utils.release_application_port(instance)
output["reserved_ports"] = "released"
# remove app config
utils.remove_application_config(app_name)
# remove from node app list
utils.remove_app_from_node_app_list(app_name)
# signal supervisor to update
p = Popen(["supervisorctl", "update"], stdout=PIPE, stderr=PIPE)
p_out, p_err = p.stdout.read().strip(), p.stderr.read().strip()
output["supervisorctl_out"] = p_out
output["supervisorctl_err"] = p_err
log.info("{0}: application removed".format(app_name))
data = {"status": "complete", "output": output, "errors": errors, "operation": "remove_application"}
return data
def deploy_app(app_name=None, package=None, build_ve=True, force_rebuild_ve=False):
"""
Deploys application
:keyword app_name: Name of application to deploy
:keyword package: Package to deploy (as tar.gz)
:keyword build_ve: Builds virtualenv for app
:keyword force_rebuild_ve: Forces a rebuild of the virtualenv (destroys existing)
"""
log = config.get_logger("deploy_app")
log.info("Deploying package {0}".format(package))
errors = {}
output = {}
tmp_deploy_dir = tempfile.mkdtemp()
# extract
try:
tf = tarfile.open(package, mode="r:gz")
tf.extractall(tmp_deploy_dir)
# look for manifest
manifest = os.path.join(tmp_deploy_dir, "manifest.json")
if os.path.exists(manifest):
mdata = json.loads(open(manifest, "r").read())
if not app_name:
if "application" in mdata:
app_name = mdata["application"]
if not app_name:
errors["deploy"] = "Unspecified application name or missing in manifest"
else:
## attempt to stop before deploy
# stop_application(app_name)
# get app config
app_config = utils.get_application_config(app_name)
if app_config:
try:
app_config = json.loads(app_config)
except:
app_config = {}
else:
app_config = {}
# create a uuid if non existent
if "uuid" not in app_config:
app_uuid = str(uuid.uuid4())
log.warn("UUID not found for {0}. Assigning {1}".format(app_name, app_uuid))
app_config["uuid"] = app_uuid
if "version" in mdata:
version = mdata["version"]
log.info("Deploying version {0}".format(version))
else:
version = None
app_config["version"] = version
if "packages" in mdata:
pkgs = mdata["packages"]
else:
pkgs = None
app_config["packages"] = pkgs
if os.path.exists(os.path.join(tmp_deploy_dir, "requirements.txt")):
reqs = os.path.join(tmp_deploy_dir, "requirements.txt")
else:
reqs = None
if "runtime" in mdata:
runtime = mdata["runtime"]
else:
runtime = None
app_config["runtime"] = runtime
if "repo_type" in mdata:
repo_type = mdata["repo_type"]
else:
repo_type = None
app_config["repo_type"] = repo_type
if "repo_url" in mdata:
repo_url = mdata["repo_url"].lower()
else:
repo_url = None
app_config["repo_url"] = repo_url
if "repo_revision" in mdata:
repo_revision = mdata["repo_revision"]
else:
repo_revision = None
app_config["repo_revision"] = repo_revision
log.debug(mdata)
## get app port
# port_reserved = False
# if 'port' in app_config:
# port = app_config['port']
# port_reserved = True
# else:
# port = utils.get_next_application_port()
# app_config['port'] = port
# get instances
if "instances" in app_config:
instances = app_config["instances"]
if settings.NODE_NAME in instances:
instances = instances[settings.NODE_NAME]
else:
instances = []
else:
instances = []
if not instances:
new_port = utils.get_next_application_port()
instances.append(new_port)
utils.reserve_application_port(new_port)
log.debug("Reserved port {0} for {1}".format(new_port, app_name))
app_config["instances"] = {}
app_config["instances"][settings.NODE_NAME] = instances
# install app
app_dir = os.path.join(settings.APPLICATION_BASE_DIR, app_name)
if not os.path.exists(app_dir):
os.makedirs(app_dir)
else:
# remove existing and re-create
shutil.rmtree(app_dir)
os.makedirs(app_dir)
install_app_data = {}
if repo_type and repo_url:
log.info("Cloning {0} from {1} using {2}".format(app_name, repo_url, repo_type))
install_app_data["repo_url"] = repo_url
if repo_type == "git":
install_app_data["repo_init"] = "Cloning with git"
p = Popen(["git", "clone", repo_url], stdout=PIPE, stderr=PIPE, cwd=app_dir)
os.waitpid(p.pid, 0)
elif repo_type == "hg":
install_app_data["repo_init"] = "Cloning with mercurial"
p = Popen(["hg", "clone", repo_url], stdout=PIPE, stderr=PIPE, cwd=app_dir)
os.waitpid(p.pid, 0)
else:
log.error("Unknown repo type: {0}".format(repo_type))
p = None
if p:
p_out, p_err = p.stdout, p.stderr
install_app_data["repo_out"] = p_out.read()
install_app_data["repo_err"] = p_out.read()
# checkout revision if needed
if repo_revision:
log.info("{0}: checking out revision {1}".format(app_name, repo_revision))
if repo_type == "git":
p = Popen(["git", "checkout", repo_revision], stdout=PIPE, stderr=PIPE, cwd=app_dir)
elif repo_type == "hg":
p = Popen(["hg", "checkout", repo_revision], stdout=PIPE, stderr=PIPE, cwd=app_dir)
else:
log.error("{0}: Unknown repo type: {0}".format(app_name, repo_type))
p = None
if p:
os.waitpid(p.pid, 0)
else:
log.debug("{0}: installing application".format(app_name))
app_dir_target = os.path.join(app_dir, app_name)
shutil.copytree(tmp_deploy_dir, app_dir_target)
output["install_app"] = install_app_data
# install ve
if build_ve:
output["install_virtualenv"] = install_virtualenv(
application=app_name, packages=pkgs, requirements=reqs, runtime=runtime, force=force_rebuild_ve
)
# update app config
utils.update_application_config(app_name, app_config)
# configure supervisor
output["configure_supervisor"] = configure_supervisor(application=app_name)
output["configure_webserver"] = configure_webserver(application=app_name)
# restart app
restart_application(app_name)
else:
log.error("Missing package manifest")
errors["deploy"] = "missing package manifest"
except Exception, e:
traceback.print_exc()
log.error("Deploy: {0}".format(traceback.format_exc()))
errors["deploy"] = str(e)